Projet

Général

Profil

Mentiodns » Historique » Version 39

sacha, 14/07/2018 18:20

1 1 sacha
h1. Mentiodns
2
3
Validation des DNS à partir d'une liste sur un unbound et un DNS grand FAI et comparaison des résultats ;)
4
5 4 sacha
h2. Noeud actifs
6
7
|_. Nom |_. Bloc |
8 23 sacha
| dam | domain_names.com_sortedaf|
9
| dam | domain_names.com_sortedag|
10
| dam | domain_names.com_sortedah|
11
| dam | domain_names.com_sortedai|
12 24 sacha
| mezzanine | domain_names.com_sortedad |
13
| mezzanine | domain_names.com_sortedao |
14
| mezzanine | domain_names.com_sortedap |
15
| mezzanine | domain_names.com_sortedaq |
16
| 1000i100 | domain_names.com_sortedab |
17
| 1000i100 | domain_names.com_sortedaj |
18
| 1000i100 | domain_names.com_sortedak |
19
| 1000i100 | domain_names.com_sortedal |
20 36 sacha
| sacha | -domain_names.org_sortedaa- |
21
| sacha | -domain_names.org_sortedab- |
22 39 sacha
| sacha | -domain_names.org_sortedac- |
23
| sacha | -domain_names.org_sortedad- |
24 37 sacha
| sacha | domain_names.net_sortedac |
25 38 sacha
| sacha | domain_names.net_sortedad |
26
| sacha | domain_names.net_sortedae |
27 24 sacha
| tazi | domain_names.com_sortedac |
28
| tazi | domain_names.com_sortedae |
29
| tazi | domain_names.com_sortedam |
30
| tazi | domain_names.com_sortedan |
31 34 sacha
| louisl | domain_names.com_sortedar |
32
| louisl | domain_names.org_sortedae  |
33
| louisl | domain_names.org_sortedaf |
34
| louisl | domain_names.net_sortedaa |
35
| louisl | domain_names.net_sortedab |
36 18 sacha
37 35 sacha
h2. Traités
38
39
domain_names.org_sortedaa 2000000 
40
domain_names.org_sortedab 2000000 
41
42
43 4 sacha
h2. Mentio
44
45 27 sacha
h3. mentio-check6
46
47 19 sacha
Packages: curl dig python socat tmux unbound whois 
48
49 4 sacha
<pre>
50
#-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+--+-#
51
#  MENTIODNS : Check for lying DNS (France)  #
52
#--------------------------------------------#
53 28 sacha
#  Version 1.6.1 date bug
54 16 sacha
#  Version 1.6 - conf file                   # 
55 13 sacha
#  Version 1.5 - test Dig resolving          #
56
#  Version 1.4 - Socat SSL sending results   # 
57
#  Version 1.3 - tld optioN                  #
58 1 sacha
#  Version 1.2 - Round robin on DNS_ISP_LIST #
59 28 sacha
#                For each request            #
60 4 sacha
#  Version 1.1 - Allow resume on basename    #
61
#  Version 1.0 - Parallel process with DIG   #
62
#--------------------------------------------#
63 1 sacha
# (c) Sacha at Aquilenet.fr part of FFDN.org #
64
#--------------------------------------------#
65
66
# This shity script intend to bruteforce the ISP lying DNS Servers to identify which one
67
# is going on Ministry of Interior Blocking page and compare the IP result from your favorite DNS server
68 16 sacha
# Use this script with the following parameters 
69
# $1 MODE: client server local
70
# $2 File source: list of domain names whithout tld
71
# $3 tld: com, org, ...
72
# $4 count number (if none from zero or from count file based on file name)
73
74
75 1 sacha
# If you relanch the script it will check if it has a counter for the given file to resume
76
# Blacklisted sites in $BLACKLIST_LOG file
77 13 sacha
# Diff ip from a domain name are in $DIFF_LOG 
78 1 sacha
79 13 sacha
# 1st launch creating config file
80 1 sacha
81 14 sacha
# Copy generated certificates:
82
# FILENAME=mentio_ssl-server 
83 1 sacha
# openssl genrsa -out $FILENAME.key 1024
84 13 sacha
# openssl req -new -key $FILENAME.key -x509 -days 3653 -out $FILENAME.crt
85 1 sacha
# cat $FILENAME.key $FILENAME.crt >$FILENAME.pem
86 13 sacha
# FILENAME=mentio_ssl-client
87
# ...
88
89 16 sacha
##########################################################
90 1 sacha
HOMEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
91 13 sacha
MENTIOCONF="$HOMEDIR/mentio.conf"
92 17 sacha
# Number of parallel requests thruw dig
93
parallel=10
94 16 sacha
##########################################################
95 1 sacha
96 16 sacha
97
##########################################################
98 13 sacha
### CHECK CONFIG
99
if [ ! -f $MENTIOCONF ]; then
100
echo "=================================================================="
101
echo "MENTIODNS"
102
echo "------------------------------------------------------------------" 
103
echo "1st time configuring"
104
echo -n "IP UNBOUND ? "
105
read DNS_MY
106 1 sacha
107 13 sacha
echo 'HOMEDIR="'$HOMEDIR'"' > $MENTIOCONF
108
echo 'DNS_MY="'$DNS_MY'"' >> $MENTIOCONF
109 1 sacha
IP_PUB=`curl ifconfig.io`
110 13 sacha
ASN=`whois -h whois.cymru.com $IP_PUB |cut -d' ' -f1|sed -n "2p"`
111 1 sacha
DNS_LIST="$HOMEDIR/mentio-DNS_ISP_LIST"
112
DNS_ISP_LIST=`grep $ASN $DNS_LIST|cut -d' ' -f3-`
113
echo 'DNS_ISP_LIST="'$DNS_ISP_LIST'"' >> $MENTIOCONF
114
echo "------------------------------------------------------------------"
115
echo " CONFIGURATION FILE:"
116
echo " please check and relaunch"
117
echo "------------------------------------------------------------------"
118
cat $MENTIOCONF
119
echo "------------------------------------------------------------------"
120
exit 1
121
fi
122 16 sacha
##########################################################
123 1 sacha
124 16 sacha
125
##########################################################
126
### PARAMETERS to execute the script
127
# Mode Log export with socat "client" "server" "local"
128
129
MODE=$1
130
131
if [ $MODE == "server" ]; then
132
socat -v -u openssl-listen:65522,fork,reuseaddr,cert=mentio_ssl-server.pem,cafile=mentio_ssl-client.crt OPEN:$HOMEDIR/MENTIO-DNS_DIFF,creat,append
133
exit 1
134
fi
135
##########################################################
136
137
138
##########################################################
139
### Check if commandline parameters are less than 3
140
141 1 sacha
if [ $# -lt 3 ]; then
142
echo "=================================================================="
143
echo "MENTIODNS"
144
echo "------------------------------------------------------------------"
145 14 sacha
echo "Missing Parameter, please enter:"
146 13 sacha
echo
147 1 sacha
echo "mentio-check client|server|local filename tld (count number)"
148 13 sacha
echo
149
exit 1
150
fi
151 16 sacha
##########################################################
152 13 sacha
153 16 sacha
154
##########################################################
155
### Get parameters
156
157
# From config file
158 1 sacha
source $MENTIOCONF
159 13 sacha
160 16 sacha
# From command line
161 1 sacha
# $2 DNS source file name
162 13 sacha
DNS_SOURCE=$2
163 1 sacha
# $3 TLD name (com, org...)
164 13 sacha
tld=$3
165 1 sacha
# line counter from the dns source file, nothing for auto-resuming
166
COUNT=$4
167
168 16 sacha
##########################################################
169 13 sacha
### SOCAT
170 29 sacha
SERVER="SOMEIP:65522"
171 13 sacha
SENDSOCAT="socat stdio openssl-connect:$SERVER,verify=0,cert=$HOMEDIR/mentio_ssl-client.pem,cafile=$HOMEDIR/mentio_ssl-server.crt"
172 16 sacha
##########################################################
173 13 sacha
### COLORS 
174
RED='\e[31m'
175
GREEN='\e[32m'
176
YELLOW='\e[33m'
177 1 sacha
GRAY='\e[90m'
178
NC='\033[0m' # No Color
179 16 sacha
##########################################################
180
### Various variables
181 13 sacha
DNS_SOURCE_BASENAME=`basename $DNS_SOURCE`
182 1 sacha
DIFF_LOG="$HOMEDIR/DNS_DIFF"
183
BLACKLIST_LOG="$HOMEDIR/DNS_BLACKLISTED"
184 13 sacha
lines=`wc -l $DNS_SOURCE|awk -F " " '{print $1}'`
185 4 sacha
countfile="$HOMEDIR/DNS_Count-$DNS_SOURCE_BASENAME"
186 16 sacha
##########################################################
187
### Dig parameters
188 1 sacha
DIG_FAST="+nodnssec +short +timeout=1 +tries=2"
189
DIG_SLOW="+nodnssec +short +timeout=5 +tries=3 "
190 16 sacha
##########################################################
191 13 sacha
192 16 sacha
193
##########################################################
194
### Generate list for dig: round robin from dns list
195
### Like (@DNS-server domain) x parallel 
196 1 sacha
_check(){
197
i=0
198 13 sacha
url=""
199
while [ $i -lt $parallel ]
200 1 sacha
do
201
n=`expr $count + $i`
202
ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
203
url="$url @$ISP_DNS `awk -v n="${n}" 'NR==n {print;exit}' $DNS_SOURCE`.$tld"
204
i=`expr $i + 1`
205
done
206
}
207 16 sacha
##########################################################
208 1 sacha
209
210 16 sacha
##########################################################
211
### Counter: create one if not existing, use existing instead
212 1 sacha
if [ -z $COUNT ]; then
213 28 sacha
        if [ -f $countfile ]; then
214
        count=`cat $countfile`
215
        else
216
        count=0
217
        echo $count > $countfile
218
        fi
219 13 sacha
else count=$COUNT
220
echo $count > $countfile
221 1 sacha
fi
222 4 sacha
##########################################################
223 16 sacha
224 4 sacha
225 16 sacha
##########################################################
226
### MAIN LOOP
227
228
while [ "$count" != "$lines" ]; do
229 4 sacha
echo $count > $countfile
230
_check
231 28 sacha
dateus=`date +%Y%m%d-%H%M%S`
232 13 sacha
site="$url"
233
echo "-------------------------------------------------------------------------------"
234
echo "#$count $dateus SITE:$site"
235
if nomentio=`dig @$DNS_MY $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio" ]; then
236 28 sacha
        echo -e "$GRAY Unknown zone $site $NC" 
237 13 sacha
fi
238
if mentio=`dig $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio" ]; then
239 28 sacha
        echo -e "$GRAY Unknown zone $site $NC"
240 4 sacha
fi
241 1 sacha
242 4 sacha
if [ -n "$nomentio" ] && [ -n "$mentio" ]; then
243 13 sacha
    if [ "$nomentio" != "$mentio" ]; then
244 28 sacha
        for i in $site; do
245
                if nomentio1=`dig $DIG_FAST @$DNS_MY $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio1" ]; then
246
                        echo -e "$GRAY Unknown zone $i $NC"
247
                fi
248
                ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
249
                if mentio1=`dig $DIG_FAST @$ISP_DNS $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio1" ]; then
250
                        echo -e "$GRAY Unknown zone $i $NC"
251
                fi
252
                        if [ "$nomentio1" != "$mentio1" ]; then
253
                                if [[ $mentio1 == 90.85.* ]]; then
254
                                        if [ $MODE == "client" ]; then
255
                                                echo "!!! $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC" | $SENDSOCAT
256
                                        fi
257
                                        echo -e "$RED !!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
258
                                        echo "!!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $BLACKLIST_LOG
259
                                else
260
                                                if [ $MODE == "client" ]; then
261
                                                        echo ">>> $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" | $SENDSOCAT
262
                                                fi                   
263
                                        echo -e "$YELLOW >>> SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
264
                                        echo ">>> $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $DIFF_LOG
265
                                fi
266
                        fi
267
        done
268 1 sacha
    else
269 13 sacha
    echo -e "$GREEN#$count  SITE:$site  $NC"
270 3 sacha
    fi
271 1 sacha
fi
272
273
count=`expr $count + $parallel`
274
275
done
276 14 sacha
##########################################################
277 28 sacha
278 17 sacha
279 14 sacha
280 7 sacha
281 2 sacha
</pre>
282 1 sacha
283 27 sacha
h3. mentio-DNS_ISP_LIST
284 1 sacha
285 27 sacha
<pre>
286
Bouygues 5410 194.158.122.10 194.158.122.15
287
Free 12322 212.27.40.240 212.27.40.241 212.27.40.244 212.27.40.245
288
Orange 3215 80.10.246.1 80.10.246.2 80.10.246.3 80.10.246.5 80.10.246.7 80.10.246.129 80.10.246.130 80.10.246.132 80.10.246.134 80.10.246.136 81.253.149.1 81.253.149.2 81.253.149.6 81.253.149.9 81.253.149.10
289
SFR 15557 109.0.66.10 109.0.66.20
290
</pre>
291 1 sacha
292 27 sacha
h3. mentio-monitor
293 1 sacha
294 27 sacha
<pre>
295
hosts="mentio-HOSTS"
296
hosts_tmp="/tmp/mentio-HOSTS.tmp"
297
rm -rf $hosts
298 1 sacha
299
300 27 sacha
GREEN='\e[32m'
301
RED='\e[31m'
302
NC='\033[0m' # No Color  
303 1 sacha
304 27 sacha
while true; do
305
        timeout 300 tail -n 0 -f MENTIO-DNS_DIFF | cut -d ' ' -f3|sort|uniq> $hosts_tmp
306
        mentiohosts=`cat $hosts_tmp`
307
        updates=`echo $mentiohosts|xargs -n 1`
308
        for i in $updates; do
309
                exists=`grep "$i" $hosts`
310
                if [ -z "$exists" ]; then
311
                        echo "ADD $i"
312
                        echo "$i"  >> $hosts 
313
                        sort -o $hosts $hosts
314
                        name=`echo $i|cut -d "." -f1`
315
                fi
316
        done
317 8 sacha
318 27 sacha
dateus=`date +%Y%m%d-%H%M%S`
319
diffs=`diff --side-by-side --suppress-common-lines $hosts $hosts_tmp`
320
echo "======================================================================"
321
if [ -z "$diffs" ]; then
322
 echo -e "$GREEN $dateus - ALL HOSTS UP: $NC"
323
 echo -e "$GREEN `cat $hosts |xargs |sort` $NC"  
324
else
325
 echo -e "$RED $dateus - MISSING HOST: $NC"
326
 echo -e "$RED $diffs $NC"
327
fi
328
done
329
</pre>
330 1 sacha
331 27 sacha
h2. Test (valide au 14/06/18)
332 1 sacha
333 27 sacha
 dig +short shahamat1.com
334
 90.85.16.52
335 1 sacha
336
h2. Vigies de la neutralité
337
338 18 sacha
https://ooni.torproject.org
339
https://respectmynet.eu
340
341
h2. Cadre légal
342
343
https://www.legifrance.gouv.fr/affichTexte.do?cidTexte=JORFTEXT000000801164&fastPos;=2&fastReqId;=606073666&categorieLien;=cid&oldAction;=rechTexte#LEGIARTI000029756525        
344
le décret https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=LEGITEXT000030315036&dateTexte;=20150305&categorieLien;=cid#LEGITEXT000030315036  
345
https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=JORFTEXT000030195477&dateTexte;=20180619
346
347
h2. Unbound
348
349
<pre>
350
server:
351
 verbosity: 1
352
 interface: 127.0.0.1
353
 do-ip4: yes
354
 do-ip6: no
355
 do-udp: yes
356
 do-tcp: no
357
 access-control: 127.0.0.0/8 allow 
358
 access-control: 0.0.0.0/0 refuse
359 15 sacha
 logfile: /var/log/unbound
360
 hide-identity: yes
361
 hide-version: yes
362
 harden-glue: yes
363
 use-caps-for-id: yes
364
 do-not-query-localhost: yes
365
</pre>
366
367 30 sacha
h2. Lancement
368
369 31 sacha
 tmux new-session -s foo -d "bash mentio-check6 client domain_names.com_sortedac com" \; split-window -h "bash mentio-check6 client domain_names.com_sortedae com" \; split-window -v "bash mentio-check6 client domain_names.com_sortedam com"\; selectp -t 0 \; split-window -v "bash mentio-check6 client domain_names.com_sortedan com"
370 15 sacha
371
h2. Ansible divers
372
373
Copy file:
374
375
 ansible mentio -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
376
377
378
Copy file single host:
379
380
 ansible mentio --limit dam -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
381
382
ansible-playbook /etc/ansible/playbooks/mentio.yml 
383
384
385
<pre>
386
---
387
388
- hosts: mentio 
389
  sudo: no
390
  tasks:
391
    - name: copyfiles 
392
      copy:
393
        src: "{{ item.src }}"
394
        dest: "{{ item.dest }}"
395
      with_items:
396 1 sacha
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-check6',dest: '~/MENTIODNS/' }
397
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-DNS_ISP_LIST',dest: '~/MENTIODNS/' }
398
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.crt',dest: '~/MENTIODNS/' }
399
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.key',dest: '~/MENTIODNS/' }
400
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.pem',dest: '~/MENTIODNS/' }
401
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-server.crt',dest: '~/MENTIODNS/' }
402
403
404
</pre>