Aquilenet » Services

Searx » Historique » Révision 9

Révision 8 (sacha, 17/05/2018 12:27) → Révision 9/15 (sacha, 10/03/2019 12:01)

# h1. Searx 

 ## Installation 

 https://github.com/asciimoo/searx   
 
 https://asciimoo.github.io/searx/ 

 Installation: https://asciimoo.github.io/searx/dev/install/installation.html 

     

 <pre> 
 a2enmod remoteip 
 </pre> 

 /etc/apache2/conf-available/remoteip.conf 

     
 <pre> 
 RemoteIPHeader X-Forwarded-For 
     
 RemoteIPTrustedProxy 127.0.0.1 ::1 

     
 </pre>  

 <pre> 
 a2enconf remoteip 
     
 service apache2 reload 
 </pre> 

 /etc/apache2/sites-available/searx.aquilenet.fr 

     

 <pre> 
 <VirtualHost *:80> 
             
         ServerName searx.aquilenet.fr 
             
         DocumentRoot /srv/www/aquilenet.fr/searx 
             
         <Directory> /srv/www/aquilenet.fr/searx> 
     
 #         RewriteEngine On 
     
 #         RewriteCond %{HTTPS} !=on 
     
 #         RewriteRule ^/?(.*) https://pad.aquilenet.fr/$1 [R,L] 
             
         Redirect permanent / https://searx.aquilenet.fr/ 
             
         Require all granted 
             
         </Directory> 
             
         Alias /.well-known/acme-challenge /srv/letsencrypt/challenges/searx.aquilenet.fr 
             
         <Directory /srv/letsencrypt/challenges/searx.aquilenet.fr> 
                 
             Require all granted 
             
         </Directory> 
     
 </VirtualHost> 
    
     

 <VirtualHost *:443> 
             
         DocumentRoot /srv/www/aquilenet.fr/searx 
             
         ServerName searx.aquilenet.fr 
             
         AllowEncodedSlashes On 
    
             

         SSLEngine on 
             
         SSLCompression off 
             
         SSLCipherSuite "HIGH:!aNULL:!MD5:!3DES:!CAMELLIA:!AES128" 
             
         SSLHonorCipherOrder on 
             
         SSLProtocol TLSv1.2 
             
         #SSLCertificateFile /etc/letsencrypt/live/searx.aquilenet.fr/fullchain.pem 
             
         #SSLCertificateKeyFile /etc/letsencrypt/live/searx.aquilenet.fr/privkey.pem 
             
         SSLCertificateFile /srv/letsencrypt/pem/searx.aquilenet.fr.pem 
             
         SSLCertificateKeyFile /srv/letsencrypt/private/searx.aquilenet.fr.key 
     
 #         SSLOpenSSLConfCmd DHParameters "/etc/ssl/private/dhparams_4096.pem" 
    
             

         ErrorLog /var/log/apache2/searx.aqln.error.log 
             
         LogLevel warn 
             
         CustomLog /var/log/apache2/searx.aqln.access.log combined 
    
             

         <FilesMatch \.xml$> 
             
         SetEnv no-gzip 1 
             
         </FilesMatch> 
    
             

         <Proxy http://127.0.0.1:4004/*> 
             
         Allow from all 
             
         </Proxy> 
    
             

         <Location /> 
             
         Options FollowSymlinks Indexes 
             
         ProxyPass http://127.0.0.1:4004/ 
             
         ProxyPassReverse http://127.0.0.1:4004/ 
             
         SetHandler uwsgi-handler 
             
         uWSGISocket /run/uwsgi/app/searx/socket 
    
             

         </location> 
             
         <Location /.well-known> 
             
         SetHandler none 
             
         </location> 
    
             

         <Directory /> 
             
         Options FollowSymLinks 
             
         AllowOverride None 
             
         Require all granted 
             
         </Directory> 
    
             

         Alias /.well-known/acme-challenge /srv/letsencrypt/challenges/searx.aquilenet.fr 
             
         <Directory /srv/letsencrypt/challenges/searx.aquilenet.fr> 
                 
             Require all granted 
             
         </Directory> 
    
     

 </VirtualHost> 

 </pre> 

 /etc/uwsgi/apps-available/searx.ini 

     

 <pre> 
 [uwsgi] 
     
 # Quel est l'utilisateur qui fera tourner le code 
     
 uid = searx 
     
 gid = searx 
    
     

 # No log + la vie privée = <3 
     
 disable-logging = true 
    
     

 # Nombre de workers (habituellement, on met le nombre de processeurs de la machine) 
     
 workers = 4 
    
     

 # Quels sont les droits sur le socket créé 
     
 chmod-socket = 666 
    
     

 # Plugin à utiliser et configuration de l'interpréteur 
     
 single-interpreter = true 
     
 master = true 
     
 plugin = python 
    
     

 # Module à importer 
     
 module = searx.webapp 
    
     

 #base = /srv/www/aquilenet.fr/searx 
    
     

 # Chemin du virtualenv 
     
 virtualenv = /srv/www/aquilenet.fr/searx/searx-ve/ 
     
 pythonpath = /srv/www/aquilenet.fr/searx/searx/ 
     
 #chdir = /srv/www/aquilenet.fr/searx/searx/ 
    
     

 #callable = app 
    
     

 # Socket 
     
 #socket = /run/uwsgi/app/searx/socket 
    
     

 #add-header = Content-Security-Policy: default-src 'self' 
     
 #add-header = X-Content-Security-Policy: default-src 'self' 
     
 #add-header = X-WebKit-CSP: default-src 'self' 
     
 #add-header = X-Content-Type-Options: nosniff 
     
 #add-header = X-XSS-Protection: 1; mode=block 
     
 #add-header = X-Frame-Options: DENY  
      
 #add-header = Strict-Transport-Security: max-age=631138519; includeSubDomains 
    
     

 #filtron 
     
 http = 127.0.0.1:8888 
 </pre> 

 ## h2. Anti bot   
 
 Pour éviter de se faire pourrir par les bots: https://asciimoo.github.io/searx/admin/filtron.html 

     

 <pre> 
 cat /srv/www/aquilenet.fr/searx/gocode/filtron/rules.json  
      
 [ 
     { 
         { 
             "name": "search request", 
             
         "filters": ["Param:q", "Path=^(/|/search)$"], 
             
         "interval": 60, 
             
         "limit": 10, 
             
         "actions": [{"name": "log"}], 
             
         "subrules": [ 
             { 
                 { 
                     "name": "roboagent limit", 
                     
                 "interval": 60,  
                      
                 "limit": 10, 
                     
                 "filters": ["Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"], 
                     
                 "actions": [ 
                         
                     {"name": "block", 
                          
                      "params": {"message": "Rate limit exceeded"}} 
                     
                 ] 
                 
             }, 
             { 
                 { 
                     "name": "botlimit", 
                     
                 "limit": 0, 
                     
                 "stop": true, 
                     
                 "filters": ["Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"], 
                     
                 "actions": [ 
                         
                     {"name": "block", 
                          
                      "params": {"message": "Rate limit exceeded"}} 
                     
                 ] 
                 
             }, 
             { 
                 { 
                     "name": "IP limit", 
                     
                 "interval": 60, 
                     
                 "limit": 10, 
                     
                 "stop": true, 
                     
                 "aggregations": ["Header:X-Forwarded-For"], 
                     
                 "actions": [ 
                         
                     {"name": "block", 
                          
                      "params": {"message": "Rate limit exceeded"}} 
                     
                 ] 
                 
             }, 
             { 
                 { 
                     "name": "rss/json limit", 
                     
                 "interval": 60, 
                     
                 "limit": 10, 
                     
                 "stop": true, 
                     
                 "filters": ["Param:format=(csv|json|rss)"], 
                     
                 "actions": [ 
                         
                     {"name": "block", 
                          
                      "params": {"message": "Rate limit exceeded"}} 
                     
                 ] 
                 
             }, 
             { 
                 { 
                     "name": "useragent limit", 
                     
                 "interval": 60, 
                     
                 "limit": 10, 
                     
                 "aggregations": ["Header:User-Agent"], 
                     
                 "actions": [ 
                         
                     {"name": "block", 
                          
                      "params": {"message": "Rate limit exceeded"}} 
                     
                 ] 
                 
             } 
             
         ] 
         
     } 
     
 ] 
 </pre> 

 creation d'un beuk-systemd service filtron 

     

 <pre> 
 /etc/systemd/system/filtron.service 
     
 [Unit] 
     
 Description=Filtron anti flood for searx Daemon 
     
 After=network-online.target 
    
     
 
 [Service] 
     
 Type=simple 
    
     
 
 User=searx 
     
 Group=searx 
     
 UMask=007 
    
     
 
 ExecStart=/srv/www/aquilenet.fr/searx/gocode/filtron/bin/filtron -rules /srv/www/aquilenet.fr/searx/gocode/filtron/rules.json 
    
     
 
 Restart=on-failure 
    
     
 
 # Configures the time to wait before service is stopped forcefully. 
     
 TimeoutStopSec=300 
    
     
 
 [Install] 
     
 WantedBy=multi-user.target 
 </pre> 

 Un peu de cli systemd: 

     
 <pre> 
 systemctl daemon-reload 
     
 systemctl enable filtron.service 
     
 systemctl start filtron 
     
 systemctl status filtron 


 ## Mise à jour 
 </pre> 

 cd /srv/www/aquilenet.fr/searx/searx 
 sudo -u searx -i 
 . ./searx-ve/bin/activate 
 git stash 
 git pull origin master 
 git stash apply 
 ./manage.sh update_packages 
 sudo service uwsgi restart