Aquilenet » Services

h1. Searx

https://github.com/asciimoo/searx

https://asciimoo.github.io/searx/

Installation: https://asciimoo.github.io/searx/dev/install/installation.html

<pre>

a2enmod remoteip

</pre>

/etc/apache2/conf-available/remoteip.conf

<pre>

RemoteIPHeader X-Forwarded-For

RemoteIPTrustedProxy 127.0.0.1 ::1

</pre> 

<pre>

a2enconf remoteip

service apache2 reload

</pre>

/etc/apache2/sites-available/searx.aquilenet.fr

<pre>

<VirtualHost *:80>

        ServerName searx.aquilenet.fr

        DocumentRoot /srv/www/aquilenet.fr/searx

        <Directory> /srv/www/aquilenet.fr/searx>

#       RewriteEngine On

#       RewriteCond %{HTTPS} !=on

#       RewriteRule ^/?(.*) https://pad.aquilenet.fr/$1 [R,L]

        Redirect permanent / https://searx.aquilenet.fr/

        Require all granted

        </Directory>

        Alias /.well-known/acme-challenge /srv/letsencrypt/challenges/searx.aquilenet.fr

        <Directory /srv/letsencrypt/challenges/searx.aquilenet.fr>

            Require all granted

        </Directory>

</VirtualHost>

<VirtualHost *:443>

        DocumentRoot /srv/www/aquilenet.fr/searx

        ServerName searx.aquilenet.fr

        AllowEncodedSlashes On

        SSLEngine on

        SSLCompression off

        SSLCipherSuite "HIGH:!aNULL:!MD5:!3DES:!CAMELLIA:!AES128"

        SSLHonorCipherOrder on

        SSLProtocol TLSv1.2

        #SSLCertificateFile /etc/letsencrypt/live/searx.aquilenet.fr/fullchain.pem

        #SSLCertificateKeyFile /etc/letsencrypt/live/searx.aquilenet.fr/privkey.pem

        SSLCertificateFile /srv/letsencrypt/pem/searx.aquilenet.fr.pem

        SSLCertificateKeyFile /srv/letsencrypt/private/searx.aquilenet.fr.key

#       SSLOpenSSLConfCmd DHParameters "/etc/ssl/private/dhparams_4096.pem"

        ErrorLog /var/log/apache2/searx.aqln.error.log

        LogLevel warn

        CustomLog /var/log/apache2/searx.aqln.access.log combined

        <FilesMatch \.xml$>

        SetEnv no-gzip 1

        </FilesMatch>

        <Proxy http://127.0.0.1:4004/*>

        Allow from all

        </Proxy>

        <Location />

        Options FollowSymlinks Indexes

        ProxyPass http://127.0.0.1:4004/

        ProxyPassReverse http://127.0.0.1:4004/

        SetHandler uwsgi-handler

        uWSGISocket /run/uwsgi/app/searx/socket

        </location>

        <Location /.well-known>

        SetHandler none

        </location>

        <Directory />

        Options FollowSymLinks

        AllowOverride None

        Require all granted

        </Directory>

        Alias /.well-known/acme-challenge /srv/letsencrypt/challenges/searx.aquilenet.fr

        <Directory /srv/letsencrypt/challenges/searx.aquilenet.fr>

            Require all granted

        </Directory>

</VirtualHost>

</pre>

/etc/uwsgi/apps-available/searx.ini

<pre>

[uwsgi]

# Quel est l'utilisateur qui fera tourner le code

uid = searx

gid = searx

# No log + la vie privée = <3

disable-logging = true

# Nombre de workers (habituellement, on met le nombre de processeurs de la machine)

workers = 4

# Quels sont les droits sur le socket créé

chmod-socket = 666

# Plugin à utiliser et configuration de l'interpréteur

single-interpreter = true

master = true

plugin = python

# Module à importer

module = searx.webapp

#base = /srv/www/aquilenet.fr/searx

# Chemin du virtualenv

virtualenv = /srv/www/aquilenet.fr/searx/searx-ve/

pythonpath = /srv/www/aquilenet.fr/searx/searx/

#chdir = /srv/www/aquilenet.fr/searx/searx/

#callable = app

# Socket

#socket = /run/uwsgi/app/searx/socket

#add-header = Content-Security-Policy: default-src 'self'

#add-header = X-Content-Security-Policy: default-src 'self'

#add-header = X-WebKit-CSP: default-src 'self'

#add-header = X-Content-Type-Options: nosniff

#add-header = X-XSS-Protection: 1; mode=block

#add-header = X-Frame-Options: DENY 

#add-header = Strict-Transport-Security: max-age=631138519; includeSubDomains

#filtron

http = 127.0.0.1:8888

</pre>

h2. Anti bot

Pour éviter de se faire pourrir par les bots: https://asciimoo.github.io/searx/admin/filtron.html

<pre>

cat /srv/www/aquilenet.fr/searx/gocode/filtron/rules.json 

[

    {

        "name": "search request",

        "filters": ["Param:q", "Path=^(/|/search)$"],

        "interval": 60,

        "limit": 10,

        "actions": [{"name": "log"}],

        "subrules": [

            {

                "name": "roboagent limit",

                "interval": 60, 

                "limit": 10,

                "filters": ["Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"],

                "actions": [

                    {"name": "block",

                     "params": {"message": "Rate limit exceeded"}}

                ]

            },

            {

                "name": "botlimit",

                "limit": 0,

                "stop": true,

                "filters": ["Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"],

                "actions": [

                    {"name": "block",

                     "params": {"message": "Rate limit exceeded"}}

                ]

            },

            {

                "name": "IP limit",

                "interval": 60,

                "limit": 10,

                "stop": true,

                "aggregations": ["Header:X-Forwarded-For"],

                "actions": [

                    {"name": "block",

                     "params": {"message": "Rate limit exceeded"}}

                ]

            },

            {

                "name": "rss/json limit",

                "interval": 60,

                "limit": 10,

                "stop": true,

                "filters": ["Param:format=(csv|json|rss)"],

                "actions": [

                    {"name": "block",

                     "params": {"message": "Rate limit exceeded"}}

                ]

            },

            {

                "name": "useragent limit",

                "interval": 60,

                "limit": 10,

                "aggregations": ["Header:User-Agent"],

                "actions": [

                    {"name": "block",

                     "params": {"message": "Rate limit exceeded"}}

                ]

            }

        ]

    }

]

</pre>

creation d'un beuk-systemd service filtron

<pre>

/etc/systemd/system/filtron.service

[Unit]

Description=Filtron anti flood for searx Daemon

After=network-online.target

[Service]

Type=simple

User=searx

Group=searx

UMask=007

ExecStart=/srv/www/aquilenet.fr/searx/gocode/filtron/bin/filtron -rules /srv/www/aquilenet.fr/searx/gocode/filtron/rules.json

Restart=on-failure

# Configures the time to wait before service is stopped forcefully.

TimeoutStopSec=300

[Install]

WantedBy=multi-user.target

</pre>

Un peu de cli systemd:

<pre>

systemctl daemon-reload

systemctl enable filtron.service

systemctl start filtron

systemctl status filtron

</pre>