Searx¶

Installation¶

https://github.com/asciimoo/searx

https://asciimoo.github.io/searx/

Installation: https://asciimoo.github.io/searx/dev/install/installation.html

a2enmod remoteip

/etc/apache2/conf-available/remoteip.conf

RemoteIPHeader X-Forwarded-For
RemoteIPTrustedProxy 127.0.0.1 ::1

a2enconf remoteip
service apache2 reload

/etc/apache2/sites-available/searx.aquilenet.fr

<VirtualHost *:80>
        ServerName searx.aquilenet.fr
        DocumentRoot /srv/www/aquilenet.fr/searx
        <Directory> /srv/www/aquilenet.fr/searx>
#       RewriteEngine On
#       RewriteCond %{HTTPS} !=on
#       RewriteRule ^/?(.*) https://pad.aquilenet.fr/$1 [R,L]
        Redirect permanent / https://searx.aquilenet.fr/
        Require all granted
        </Directory>
        Alias /.well-known/acme-challenge /srv/letsencrypt/challenges/searx.aquilenet.fr
        <Directory /srv/letsencrypt/challenges/searx.aquilenet.fr>
            Require all granted
        </Directory>
</VirtualHost>

<VirtualHost *:443>
        DocumentRoot /srv/www/aquilenet.fr/searx
        ServerName searx.aquilenet.fr
        AllowEncodedSlashes On

        SSLEngine on
        SSLCompression off
        SSLCipherSuite "HIGH:!aNULL:!MD5:!3DES:!CAMELLIA:!AES128"
        SSLHonorCipherOrder on
        SSLProtocol TLSv1.2
        #SSLCertificateFile /etc/letsencrypt/live/searx.aquilenet.fr/fullchain.pem
        #SSLCertificateKeyFile /etc/letsencrypt/live/searx.aquilenet.fr/privkey.pem
        SSLCertificateFile /srv/letsencrypt/pem/searx.aquilenet.fr.pem
        SSLCertificateKeyFile /srv/letsencrypt/private/searx.aquilenet.fr.key
#       SSLOpenSSLConfCmd DHParameters "/etc/ssl/private/dhparams_4096.pem"

        ErrorLog /var/log/apache2/searx.aqln.error.log
        LogLevel warn
        CustomLog /var/log/apache2/searx.aqln.access.log combined

        <FilesMatch \.xml$>
        SetEnv no-gzip 1
        </FilesMatch>

        <Proxy http://127.0.0.1:4004/*>
        Allow from all
        </Proxy>

        <Location />
        Options FollowSymlinks Indexes
        ProxyPass http://127.0.0.1:4004/
        ProxyPassReverse http://127.0.0.1:4004/
        SetHandler uwsgi-handler
        uWSGISocket /run/uwsgi/app/searx/socket

        </location>
        <Location /.well-known>
        SetHandler none
        </location>

        <Directory />
        Options FollowSymLinks
        AllowOverride None
        Require all granted
        </Directory>

        Alias /.well-known/acme-challenge /srv/letsencrypt/challenges/searx.aquilenet.fr
        <Directory /srv/letsencrypt/challenges/searx.aquilenet.fr>
            Require all granted
        </Directory>

</VirtualHost>

/etc/uwsgi/apps-available/searx.ini

[uwsgi]
# Quel est l'utilisateur qui fera tourner le code
uid = searx
gid = searx

# No log + la vie privée = <3
disable-logging = true

# Nombre de workers (habituellement, on met le nombre de processeurs de la machine)
workers = 4

# Quels sont les droits sur le socket créé
chmod-socket = 666

# Plugin à utiliser et configuration de l'interpréteur
single-interpreter = true
master = true
plugin = python

# Module à importer
module = searx.webapp

#base = /srv/www/aquilenet.fr/searx

# Chemin du virtualenv
virtualenv = /srv/www/aquilenet.fr/searx/searx/searx-ve/
pythonpath = /srv/www/aquilenet.fr/searx/searx/
#chdir = /srv/www/aquilenet.fr/searx/searx/

#callable = app

# Socket
#socket = /run/uwsgi/app/searx/socket

#add-header = Content-Security-Policy: default-src 'self'
#add-header = X-Content-Security-Policy: default-src 'self'
#add-header = X-WebKit-CSP: default-src 'self'
#add-header = X-Content-Type-Options: nosniff
#add-header = X-XSS-Protection: 1; mode=block
#add-header = X-Frame-Options: DENY 
#add-header = Strict-Transport-Security: max-age=631138519; includeSubDomains

#filtron
http = 127.0.0.1:8888

Anti bot¶

Pour éviter de se faire pourrir par les bots: https://asciimoo.github.io/searx/admin/filtron.html

cat /srv/www/aquilenet.fr/searx/gocode/filtron/rules.json 
[
    {
        "name": "search request",
        "filters": ["Param:q", "Path=^(/|/search)$"],
        "interval": 60,
        "limit": 10,
        "actions": [{"name": "log"}],
        "subrules": [
            {
                "name": "roboagent limit",
                "interval": 60, 
                "limit": 10,
                "filters": ["Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"],
                "actions": [
                    {"name": "block",
                     "params": {"message": "Rate limit exceeded"}}
                ]
            },
            {
                "name": "botlimit",
                "limit": 0,
                "stop": true,
                "filters": ["Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"],
                "actions": [
                    {"name": "block",
                     "params": {"message": "Rate limit exceeded"}}
                ]
            },
            {
                "name": "IP limit",
                        "interval": 60,
                        "limit": 2,
                        "stop": true,
                        "aggregations": ["Header:X-Forwarded-For"],
                        "actions": [
                            {"name": "block",
                             "params": {"message": "IP-Blocked"}}
                         ]
            },
            {
                "name": "rss/json limit",
                "interval": 60,
                "limit": 2,
                "stop": true,
                "filters": ["Param:format=(csv|json|rss)"],
                "actions": [
                    {"name": "block",
                     "params": {"message": "Rate limit exceeded"}}
                ]
            },
            {
                "name": "useragent limit",
                "interval": 60,
                "limit": 10,
                "aggregations": ["Header:User-Agent"],
                "actions": [
                    {"name": "block",
                     "params": {"message": "Rate limit exceeded"}}
                ]
            }
        ]
    }
]

creation d'un beuk-systemd service filtron

/etc/systemd/system/filtron.service
[Unit]
Description=Filtron anti flood for searx Daemon
After=network-online.target

[Service]
Type=simple

User=searx
Group=searx
UMask=007

ExecStart=/srv/www/aquilenet.fr/searx/gocode/filtron/bin/filtron -rules /srv/www/aquilenet.fr/searx/gocode/filtron/rules.json

Restart=on-failure

# Configures the time to wait before service is stopped forcefully.
TimeoutStopSec=300

[Install]
WantedBy=multi-user.target

Un peu de cli systemd:

systemctl daemon-reload
systemctl enable filtron.service
systemctl start filtron
systemctl status filtron

Mise à jour¶

cd /srv/www/aquilenet.fr/searx/searx
sudo -u searx -i
. ./searx-ve/bin/activate
git stash
git pull origin master
git stash apply
./manage.sh update_packages
sudo service uwsgi restart