Projet

Général

Profil

Mentiodns » Historique » Version 40

sacha, 14/07/2018 18:21

1 1 sacha
h1. Mentiodns
2
3
Validation des DNS à partir d'une liste sur un unbound et un DNS grand FAI et comparaison des résultats ;)
4
5 4 sacha
h2. Noeud actifs
6
7
|_. Nom |_. Bloc |
8 23 sacha
| dam | domain_names.com_sortedaf|
9
| dam | domain_names.com_sortedag|
10
| dam | domain_names.com_sortedah|
11
| dam | domain_names.com_sortedai|
12 24 sacha
| mezzanine | domain_names.com_sortedad |
13
| mezzanine | domain_names.com_sortedao |
14
| mezzanine | domain_names.com_sortedap |
15
| mezzanine | domain_names.com_sortedaq |
16
| 1000i100 | domain_names.com_sortedab |
17
| 1000i100 | domain_names.com_sortedaj |
18
| 1000i100 | domain_names.com_sortedak |
19
| 1000i100 | domain_names.com_sortedal |
20 36 sacha
| sacha | -domain_names.org_sortedaa- |
21
| sacha | -domain_names.org_sortedab- |
22 39 sacha
| sacha | -domain_names.org_sortedac- |
23
| sacha | -domain_names.org_sortedad- |
24 37 sacha
| sacha | domain_names.net_sortedac |
25 38 sacha
| sacha | domain_names.net_sortedad |
26
| sacha | domain_names.net_sortedae |
27 24 sacha
| tazi | domain_names.com_sortedac |
28
| tazi | domain_names.com_sortedae |
29
| tazi | domain_names.com_sortedam |
30
| tazi | domain_names.com_sortedan |
31 34 sacha
| louisl | domain_names.com_sortedar |
32
| louisl | domain_names.org_sortedae  |
33
| louisl | domain_names.org_sortedaf |
34
| louisl | domain_names.net_sortedaa |
35
| louisl | domain_names.net_sortedab |
36 18 sacha
37 35 sacha
h2. Traités
38
39
domain_names.org_sortedaa 2000000 
40
domain_names.org_sortedab 2000000 
41 40 sacha
domain_names.org_sortedac 2000000
42
domain_names.org_sortedad 2000000
43 35 sacha
44 4 sacha
h2. Mentio
45
46 27 sacha
h3. mentio-check6
47
48 19 sacha
Packages: curl dig python socat tmux unbound whois 
49
50 4 sacha
<pre>
51
#-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+--+-#
52
#  MENTIODNS : Check for lying DNS (France)  #
53
#--------------------------------------------#
54 28 sacha
#  Version 1.6.1 date bug
55 16 sacha
#  Version 1.6 - conf file                   # 
56 13 sacha
#  Version 1.5 - test Dig resolving          #
57
#  Version 1.4 - Socat SSL sending results   # 
58
#  Version 1.3 - tld optioN                  #
59 1 sacha
#  Version 1.2 - Round robin on DNS_ISP_LIST #
60 28 sacha
#                For each request            #
61 4 sacha
#  Version 1.1 - Allow resume on basename    #
62
#  Version 1.0 - Parallel process with DIG   #
63
#--------------------------------------------#
64 1 sacha
# (c) Sacha at Aquilenet.fr part of FFDN.org #
65
#--------------------------------------------#
66
67
# This shity script intend to bruteforce the ISP lying DNS Servers to identify which one
68
# is going on Ministry of Interior Blocking page and compare the IP result from your favorite DNS server
69 16 sacha
# Use this script with the following parameters 
70
# $1 MODE: client server local
71
# $2 File source: list of domain names whithout tld
72
# $3 tld: com, org, ...
73
# $4 count number (if none from zero or from count file based on file name)
74
75
76 1 sacha
# If you relanch the script it will check if it has a counter for the given file to resume
77
# Blacklisted sites in $BLACKLIST_LOG file
78 13 sacha
# Diff ip from a domain name are in $DIFF_LOG 
79 1 sacha
80 13 sacha
# 1st launch creating config file
81 1 sacha
82 14 sacha
# Copy generated certificates:
83
# FILENAME=mentio_ssl-server 
84 1 sacha
# openssl genrsa -out $FILENAME.key 1024
85 13 sacha
# openssl req -new -key $FILENAME.key -x509 -days 3653 -out $FILENAME.crt
86 1 sacha
# cat $FILENAME.key $FILENAME.crt >$FILENAME.pem
87 13 sacha
# FILENAME=mentio_ssl-client
88
# ...
89
90 16 sacha
##########################################################
91 1 sacha
HOMEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
92 13 sacha
MENTIOCONF="$HOMEDIR/mentio.conf"
93 17 sacha
# Number of parallel requests thruw dig
94
parallel=10
95 16 sacha
##########################################################
96 1 sacha
97 16 sacha
98
##########################################################
99 13 sacha
### CHECK CONFIG
100
if [ ! -f $MENTIOCONF ]; then
101
echo "=================================================================="
102
echo "MENTIODNS"
103
echo "------------------------------------------------------------------" 
104
echo "1st time configuring"
105
echo -n "IP UNBOUND ? "
106
read DNS_MY
107 1 sacha
108 13 sacha
echo 'HOMEDIR="'$HOMEDIR'"' > $MENTIOCONF
109
echo 'DNS_MY="'$DNS_MY'"' >> $MENTIOCONF
110 1 sacha
IP_PUB=`curl ifconfig.io`
111 13 sacha
ASN=`whois -h whois.cymru.com $IP_PUB |cut -d' ' -f1|sed -n "2p"`
112 1 sacha
DNS_LIST="$HOMEDIR/mentio-DNS_ISP_LIST"
113
DNS_ISP_LIST=`grep $ASN $DNS_LIST|cut -d' ' -f3-`
114
echo 'DNS_ISP_LIST="'$DNS_ISP_LIST'"' >> $MENTIOCONF
115
echo "------------------------------------------------------------------"
116
echo " CONFIGURATION FILE:"
117
echo " please check and relaunch"
118
echo "------------------------------------------------------------------"
119
cat $MENTIOCONF
120
echo "------------------------------------------------------------------"
121
exit 1
122
fi
123 16 sacha
##########################################################
124 1 sacha
125 16 sacha
126
##########################################################
127
### PARAMETERS to execute the script
128
# Mode Log export with socat "client" "server" "local"
129
130
MODE=$1
131
132
if [ $MODE == "server" ]; then
133
socat -v -u openssl-listen:65522,fork,reuseaddr,cert=mentio_ssl-server.pem,cafile=mentio_ssl-client.crt OPEN:$HOMEDIR/MENTIO-DNS_DIFF,creat,append
134
exit 1
135
fi
136
##########################################################
137
138
139
##########################################################
140
### Check if commandline parameters are less than 3
141
142 1 sacha
if [ $# -lt 3 ]; then
143
echo "=================================================================="
144
echo "MENTIODNS"
145
echo "------------------------------------------------------------------"
146 14 sacha
echo "Missing Parameter, please enter:"
147 13 sacha
echo
148 1 sacha
echo "mentio-check client|server|local filename tld (count number)"
149 13 sacha
echo
150
exit 1
151
fi
152 16 sacha
##########################################################
153 13 sacha
154 16 sacha
155
##########################################################
156
### Get parameters
157
158
# From config file
159 1 sacha
source $MENTIOCONF
160 13 sacha
161 16 sacha
# From command line
162 1 sacha
# $2 DNS source file name
163 13 sacha
DNS_SOURCE=$2
164 1 sacha
# $3 TLD name (com, org...)
165 13 sacha
tld=$3
166 1 sacha
# line counter from the dns source file, nothing for auto-resuming
167
COUNT=$4
168
169 16 sacha
##########################################################
170 13 sacha
### SOCAT
171 29 sacha
SERVER="SOMEIP:65522"
172 13 sacha
SENDSOCAT="socat stdio openssl-connect:$SERVER,verify=0,cert=$HOMEDIR/mentio_ssl-client.pem,cafile=$HOMEDIR/mentio_ssl-server.crt"
173 16 sacha
##########################################################
174 13 sacha
### COLORS 
175
RED='\e[31m'
176
GREEN='\e[32m'
177
YELLOW='\e[33m'
178 1 sacha
GRAY='\e[90m'
179
NC='\033[0m' # No Color
180 16 sacha
##########################################################
181
### Various variables
182 13 sacha
DNS_SOURCE_BASENAME=`basename $DNS_SOURCE`
183 1 sacha
DIFF_LOG="$HOMEDIR/DNS_DIFF"
184
BLACKLIST_LOG="$HOMEDIR/DNS_BLACKLISTED"
185 13 sacha
lines=`wc -l $DNS_SOURCE|awk -F " " '{print $1}'`
186 4 sacha
countfile="$HOMEDIR/DNS_Count-$DNS_SOURCE_BASENAME"
187 16 sacha
##########################################################
188
### Dig parameters
189 1 sacha
DIG_FAST="+nodnssec +short +timeout=1 +tries=2"
190
DIG_SLOW="+nodnssec +short +timeout=5 +tries=3 "
191 16 sacha
##########################################################
192 13 sacha
193 16 sacha
194
##########################################################
195
### Generate list for dig: round robin from dns list
196
### Like (@DNS-server domain) x parallel 
197 1 sacha
_check(){
198
i=0
199 13 sacha
url=""
200
while [ $i -lt $parallel ]
201 1 sacha
do
202
n=`expr $count + $i`
203
ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
204
url="$url @$ISP_DNS `awk -v n="${n}" 'NR==n {print;exit}' $DNS_SOURCE`.$tld"
205
i=`expr $i + 1`
206
done
207
}
208 16 sacha
##########################################################
209 1 sacha
210
211 16 sacha
##########################################################
212
### Counter: create one if not existing, use existing instead
213 1 sacha
if [ -z $COUNT ]; then
214 28 sacha
        if [ -f $countfile ]; then
215
        count=`cat $countfile`
216
        else
217
        count=0
218
        echo $count > $countfile
219
        fi
220 13 sacha
else count=$COUNT
221
echo $count > $countfile
222 1 sacha
fi
223 4 sacha
##########################################################
224 16 sacha
225 4 sacha
226 16 sacha
##########################################################
227
### MAIN LOOP
228
229
while [ "$count" != "$lines" ]; do
230 4 sacha
echo $count > $countfile
231
_check
232 28 sacha
dateus=`date +%Y%m%d-%H%M%S`
233 13 sacha
site="$url"
234
echo "-------------------------------------------------------------------------------"
235
echo "#$count $dateus SITE:$site"
236
if nomentio=`dig @$DNS_MY $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio" ]; then
237 28 sacha
        echo -e "$GRAY Unknown zone $site $NC" 
238 13 sacha
fi
239
if mentio=`dig $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio" ]; then
240 28 sacha
        echo -e "$GRAY Unknown zone $site $NC"
241 4 sacha
fi
242 1 sacha
243 4 sacha
if [ -n "$nomentio" ] && [ -n "$mentio" ]; then
244 13 sacha
    if [ "$nomentio" != "$mentio" ]; then
245 28 sacha
        for i in $site; do
246
                if nomentio1=`dig $DIG_FAST @$DNS_MY $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio1" ]; then
247
                        echo -e "$GRAY Unknown zone $i $NC"
248
                fi
249
                ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
250
                if mentio1=`dig $DIG_FAST @$ISP_DNS $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio1" ]; then
251
                        echo -e "$GRAY Unknown zone $i $NC"
252
                fi
253
                        if [ "$nomentio1" != "$mentio1" ]; then
254
                                if [[ $mentio1 == 90.85.* ]]; then
255
                                        if [ $MODE == "client" ]; then
256
                                                echo "!!! $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC" | $SENDSOCAT
257
                                        fi
258
                                        echo -e "$RED !!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
259
                                        echo "!!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $BLACKLIST_LOG
260
                                else
261
                                                if [ $MODE == "client" ]; then
262
                                                        echo ">>> $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" | $SENDSOCAT
263
                                                fi                   
264
                                        echo -e "$YELLOW >>> SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
265
                                        echo ">>> $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $DIFF_LOG
266
                                fi
267
                        fi
268
        done
269 1 sacha
    else
270 13 sacha
    echo -e "$GREEN#$count  SITE:$site  $NC"
271 3 sacha
    fi
272 1 sacha
fi
273
274
count=`expr $count + $parallel`
275
276
done
277 14 sacha
##########################################################
278 28 sacha
279 17 sacha
280 14 sacha
281 7 sacha
282 2 sacha
</pre>
283 1 sacha
284 27 sacha
h3. mentio-DNS_ISP_LIST
285 1 sacha
286 27 sacha
<pre>
287
Bouygues 5410 194.158.122.10 194.158.122.15
288
Free 12322 212.27.40.240 212.27.40.241 212.27.40.244 212.27.40.245
289
Orange 3215 80.10.246.1 80.10.246.2 80.10.246.3 80.10.246.5 80.10.246.7 80.10.246.129 80.10.246.130 80.10.246.132 80.10.246.134 80.10.246.136 81.253.149.1 81.253.149.2 81.253.149.6 81.253.149.9 81.253.149.10
290
SFR 15557 109.0.66.10 109.0.66.20
291
</pre>
292 1 sacha
293 27 sacha
h3. mentio-monitor
294 1 sacha
295 27 sacha
<pre>
296
hosts="mentio-HOSTS"
297
hosts_tmp="/tmp/mentio-HOSTS.tmp"
298
rm -rf $hosts
299 1 sacha
300
301 27 sacha
GREEN='\e[32m'
302
RED='\e[31m'
303
NC='\033[0m' # No Color  
304 1 sacha
305 27 sacha
while true; do
306
        timeout 300 tail -n 0 -f MENTIO-DNS_DIFF | cut -d ' ' -f3|sort|uniq> $hosts_tmp
307
        mentiohosts=`cat $hosts_tmp`
308
        updates=`echo $mentiohosts|xargs -n 1`
309
        for i in $updates; do
310
                exists=`grep "$i" $hosts`
311
                if [ -z "$exists" ]; then
312
                        echo "ADD $i"
313
                        echo "$i"  >> $hosts 
314
                        sort -o $hosts $hosts
315
                        name=`echo $i|cut -d "." -f1`
316
                fi
317
        done
318 8 sacha
319 27 sacha
dateus=`date +%Y%m%d-%H%M%S`
320
diffs=`diff --side-by-side --suppress-common-lines $hosts $hosts_tmp`
321
echo "======================================================================"
322
if [ -z "$diffs" ]; then
323
 echo -e "$GREEN $dateus - ALL HOSTS UP: $NC"
324
 echo -e "$GREEN `cat $hosts |xargs |sort` $NC"  
325
else
326
 echo -e "$RED $dateus - MISSING HOST: $NC"
327
 echo -e "$RED $diffs $NC"
328
fi
329
done
330
</pre>
331 1 sacha
332 27 sacha
h2. Test (valide au 14/06/18)
333 1 sacha
334 27 sacha
 dig +short shahamat1.com
335
 90.85.16.52
336 1 sacha
337
h2. Vigies de la neutralité
338
339 18 sacha
https://ooni.torproject.org
340
https://respectmynet.eu
341
342
h2. Cadre légal
343
344
https://www.legifrance.gouv.fr/affichTexte.do?cidTexte=JORFTEXT000000801164&fastPos;=2&fastReqId;=606073666&categorieLien;=cid&oldAction;=rechTexte#LEGIARTI000029756525        
345
le décret https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=LEGITEXT000030315036&dateTexte;=20150305&categorieLien;=cid#LEGITEXT000030315036  
346
https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=JORFTEXT000030195477&dateTexte;=20180619
347
348
h2. Unbound
349
350
<pre>
351
server:
352
 verbosity: 1
353
 interface: 127.0.0.1
354
 do-ip4: yes
355
 do-ip6: no
356
 do-udp: yes
357
 do-tcp: no
358
 access-control: 127.0.0.0/8 allow 
359
 access-control: 0.0.0.0/0 refuse
360 15 sacha
 logfile: /var/log/unbound
361
 hide-identity: yes
362
 hide-version: yes
363
 harden-glue: yes
364
 use-caps-for-id: yes
365
 do-not-query-localhost: yes
366
</pre>
367
368 30 sacha
h2. Lancement
369
370 31 sacha
 tmux new-session -s foo -d "bash mentio-check6 client domain_names.com_sortedac com" \; split-window -h "bash mentio-check6 client domain_names.com_sortedae com" \; split-window -v "bash mentio-check6 client domain_names.com_sortedam com"\; selectp -t 0 \; split-window -v "bash mentio-check6 client domain_names.com_sortedan com"
371 15 sacha
372
h2. Ansible divers
373
374
Copy file:
375
376
 ansible mentio -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
377
378
379
Copy file single host:
380
381
 ansible mentio --limit dam -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
382
383
ansible-playbook /etc/ansible/playbooks/mentio.yml 
384
385
386
<pre>
387
---
388
389
- hosts: mentio 
390
  sudo: no
391
  tasks:
392
    - name: copyfiles 
393
      copy:
394
        src: "{{ item.src }}"
395
        dest: "{{ item.dest }}"
396
      with_items:
397 1 sacha
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-check6',dest: '~/MENTIODNS/' }
398
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-DNS_ISP_LIST',dest: '~/MENTIODNS/' }
399
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.crt',dest: '~/MENTIODNS/' }
400
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.key',dest: '~/MENTIODNS/' }
401
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.pem',dest: '~/MENTIODNS/' }
402
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-server.crt',dest: '~/MENTIODNS/' }
403
404
405
</pre>