Projet

Général

Profil

Mentiodns » Historique » Version 37

sacha, 12/07/2018 12:47

1 1 sacha
h1. Mentiodns
2
3
Validation des DNS à partir d'une liste sur un unbound et un DNS grand FAI et comparaison des résultats ;)
4
5 4 sacha
h2. Noeud actifs
6
7
|_. Nom |_. Bloc |
8 23 sacha
| dam | domain_names.com_sortedaf|
9
| dam | domain_names.com_sortedag|
10
| dam | domain_names.com_sortedah|
11
| dam | domain_names.com_sortedai|
12 24 sacha
| mezzanine | domain_names.com_sortedad |
13
| mezzanine | domain_names.com_sortedao |
14
| mezzanine | domain_names.com_sortedap |
15
| mezzanine | domain_names.com_sortedaq |
16
| 1000i100 | domain_names.com_sortedab |
17
| 1000i100 | domain_names.com_sortedaj |
18
| 1000i100 | domain_names.com_sortedak |
19
| 1000i100 | domain_names.com_sortedal |
20 36 sacha
| sacha | -domain_names.org_sortedaa- |
21
| sacha | -domain_names.org_sortedab- |
22 26 sacha
| sacha | domain_names.org_sortedac |
23
| sacha | domain_names.org_sortedad |
24 37 sacha
| sacha | domain_names.net_sortedac |
25 24 sacha
| tazi | domain_names.com_sortedac |
26
| tazi | domain_names.com_sortedae |
27
| tazi | domain_names.com_sortedam |
28
| tazi | domain_names.com_sortedan |
29 34 sacha
| louisl | domain_names.com_sortedar |
30
| louisl | domain_names.org_sortedae  |
31
| louisl | domain_names.org_sortedaf |
32
| louisl | domain_names.net_sortedaa |
33
| louisl | domain_names.net_sortedab |
34 18 sacha
35 35 sacha
h2. Traités
36
37
domain_names.org_sortedaa 2000000 
38
domain_names.org_sortedab 2000000 
39
40
41 4 sacha
h2. Mentio
42
43 27 sacha
h3. mentio-check6
44
45 19 sacha
Packages: curl dig python socat tmux unbound whois 
46
47 4 sacha
<pre>
48
#-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+--+-#
49
#  MENTIODNS : Check for lying DNS (France)  #
50
#--------------------------------------------#
51 28 sacha
#  Version 1.6.1 date bug
52 16 sacha
#  Version 1.6 - conf file                   # 
53 13 sacha
#  Version 1.5 - test Dig resolving          #
54
#  Version 1.4 - Socat SSL sending results   # 
55
#  Version 1.3 - tld optioN                  #
56 1 sacha
#  Version 1.2 - Round robin on DNS_ISP_LIST #
57 28 sacha
#                For each request            #
58 4 sacha
#  Version 1.1 - Allow resume on basename    #
59
#  Version 1.0 - Parallel process with DIG   #
60
#--------------------------------------------#
61 1 sacha
# (c) Sacha at Aquilenet.fr part of FFDN.org #
62
#--------------------------------------------#
63
64
# This shity script intend to bruteforce the ISP lying DNS Servers to identify which one
65
# is going on Ministry of Interior Blocking page and compare the IP result from your favorite DNS server
66 16 sacha
# Use this script with the following parameters 
67
# $1 MODE: client server local
68
# $2 File source: list of domain names whithout tld
69
# $3 tld: com, org, ...
70
# $4 count number (if none from zero or from count file based on file name)
71
72
73 1 sacha
# If you relanch the script it will check if it has a counter for the given file to resume
74
# Blacklisted sites in $BLACKLIST_LOG file
75 13 sacha
# Diff ip from a domain name are in $DIFF_LOG 
76 1 sacha
77 13 sacha
# 1st launch creating config file
78 1 sacha
79 14 sacha
# Copy generated certificates:
80
# FILENAME=mentio_ssl-server 
81 1 sacha
# openssl genrsa -out $FILENAME.key 1024
82 13 sacha
# openssl req -new -key $FILENAME.key -x509 -days 3653 -out $FILENAME.crt
83 1 sacha
# cat $FILENAME.key $FILENAME.crt >$FILENAME.pem
84 13 sacha
# FILENAME=mentio_ssl-client
85
# ...
86
87 16 sacha
##########################################################
88 1 sacha
HOMEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
89 13 sacha
MENTIOCONF="$HOMEDIR/mentio.conf"
90 17 sacha
# Number of parallel requests thruw dig
91
parallel=10
92 16 sacha
##########################################################
93 1 sacha
94 16 sacha
95
##########################################################
96 13 sacha
### CHECK CONFIG
97
if [ ! -f $MENTIOCONF ]; then
98
echo "=================================================================="
99
echo "MENTIODNS"
100
echo "------------------------------------------------------------------" 
101
echo "1st time configuring"
102
echo -n "IP UNBOUND ? "
103
read DNS_MY
104 1 sacha
105 13 sacha
echo 'HOMEDIR="'$HOMEDIR'"' > $MENTIOCONF
106
echo 'DNS_MY="'$DNS_MY'"' >> $MENTIOCONF
107 1 sacha
IP_PUB=`curl ifconfig.io`
108 13 sacha
ASN=`whois -h whois.cymru.com $IP_PUB |cut -d' ' -f1|sed -n "2p"`
109 1 sacha
DNS_LIST="$HOMEDIR/mentio-DNS_ISP_LIST"
110
DNS_ISP_LIST=`grep $ASN $DNS_LIST|cut -d' ' -f3-`
111
echo 'DNS_ISP_LIST="'$DNS_ISP_LIST'"' >> $MENTIOCONF
112
echo "------------------------------------------------------------------"
113
echo " CONFIGURATION FILE:"
114
echo " please check and relaunch"
115
echo "------------------------------------------------------------------"
116
cat $MENTIOCONF
117
echo "------------------------------------------------------------------"
118
exit 1
119
fi
120 16 sacha
##########################################################
121 1 sacha
122 16 sacha
123
##########################################################
124
### PARAMETERS to execute the script
125
# Mode Log export with socat "client" "server" "local"
126
127
MODE=$1
128
129
if [ $MODE == "server" ]; then
130
socat -v -u openssl-listen:65522,fork,reuseaddr,cert=mentio_ssl-server.pem,cafile=mentio_ssl-client.crt OPEN:$HOMEDIR/MENTIO-DNS_DIFF,creat,append
131
exit 1
132
fi
133
##########################################################
134
135
136
##########################################################
137
### Check if commandline parameters are less than 3
138
139 1 sacha
if [ $# -lt 3 ]; then
140
echo "=================================================================="
141
echo "MENTIODNS"
142
echo "------------------------------------------------------------------"
143 14 sacha
echo "Missing Parameter, please enter:"
144 13 sacha
echo
145 1 sacha
echo "mentio-check client|server|local filename tld (count number)"
146 13 sacha
echo
147
exit 1
148
fi
149 16 sacha
##########################################################
150 13 sacha
151 16 sacha
152
##########################################################
153
### Get parameters
154
155
# From config file
156 1 sacha
source $MENTIOCONF
157 13 sacha
158 16 sacha
# From command line
159 1 sacha
# $2 DNS source file name
160 13 sacha
DNS_SOURCE=$2
161 1 sacha
# $3 TLD name (com, org...)
162 13 sacha
tld=$3
163 1 sacha
# line counter from the dns source file, nothing for auto-resuming
164
COUNT=$4
165
166 16 sacha
##########################################################
167 13 sacha
### SOCAT
168 29 sacha
SERVER="SOMEIP:65522"
169 13 sacha
SENDSOCAT="socat stdio openssl-connect:$SERVER,verify=0,cert=$HOMEDIR/mentio_ssl-client.pem,cafile=$HOMEDIR/mentio_ssl-server.crt"
170 16 sacha
##########################################################
171 13 sacha
### COLORS 
172
RED='\e[31m'
173
GREEN='\e[32m'
174
YELLOW='\e[33m'
175 1 sacha
GRAY='\e[90m'
176
NC='\033[0m' # No Color
177 16 sacha
##########################################################
178
### Various variables
179 13 sacha
DNS_SOURCE_BASENAME=`basename $DNS_SOURCE`
180 1 sacha
DIFF_LOG="$HOMEDIR/DNS_DIFF"
181
BLACKLIST_LOG="$HOMEDIR/DNS_BLACKLISTED"
182 13 sacha
lines=`wc -l $DNS_SOURCE|awk -F " " '{print $1}'`
183 4 sacha
countfile="$HOMEDIR/DNS_Count-$DNS_SOURCE_BASENAME"
184 16 sacha
##########################################################
185
### Dig parameters
186 1 sacha
DIG_FAST="+nodnssec +short +timeout=1 +tries=2"
187
DIG_SLOW="+nodnssec +short +timeout=5 +tries=3 "
188 16 sacha
##########################################################
189 13 sacha
190 16 sacha
191
##########################################################
192
### Generate list for dig: round robin from dns list
193
### Like (@DNS-server domain) x parallel 
194 1 sacha
_check(){
195
i=0
196 13 sacha
url=""
197
while [ $i -lt $parallel ]
198 1 sacha
do
199
n=`expr $count + $i`
200
ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
201
url="$url @$ISP_DNS `awk -v n="${n}" 'NR==n {print;exit}' $DNS_SOURCE`.$tld"
202
i=`expr $i + 1`
203
done
204
}
205 16 sacha
##########################################################
206 1 sacha
207
208 16 sacha
##########################################################
209
### Counter: create one if not existing, use existing instead
210 1 sacha
if [ -z $COUNT ]; then
211 28 sacha
        if [ -f $countfile ]; then
212
        count=`cat $countfile`
213
        else
214
        count=0
215
        echo $count > $countfile
216
        fi
217 13 sacha
else count=$COUNT
218
echo $count > $countfile
219 1 sacha
fi
220 4 sacha
##########################################################
221 16 sacha
222 4 sacha
223 16 sacha
##########################################################
224
### MAIN LOOP
225
226
while [ "$count" != "$lines" ]; do
227 4 sacha
echo $count > $countfile
228
_check
229 28 sacha
dateus=`date +%Y%m%d-%H%M%S`
230 13 sacha
site="$url"
231
echo "-------------------------------------------------------------------------------"
232
echo "#$count $dateus SITE:$site"
233
if nomentio=`dig @$DNS_MY $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio" ]; then
234 28 sacha
        echo -e "$GRAY Unknown zone $site $NC" 
235 13 sacha
fi
236
if mentio=`dig $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio" ]; then
237 28 sacha
        echo -e "$GRAY Unknown zone $site $NC"
238 4 sacha
fi
239 1 sacha
240 4 sacha
if [ -n "$nomentio" ] && [ -n "$mentio" ]; then
241 13 sacha
    if [ "$nomentio" != "$mentio" ]; then
242 28 sacha
        for i in $site; do
243
                if nomentio1=`dig $DIG_FAST @$DNS_MY $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio1" ]; then
244
                        echo -e "$GRAY Unknown zone $i $NC"
245
                fi
246
                ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
247
                if mentio1=`dig $DIG_FAST @$ISP_DNS $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio1" ]; then
248
                        echo -e "$GRAY Unknown zone $i $NC"
249
                fi
250
                        if [ "$nomentio1" != "$mentio1" ]; then
251
                                if [[ $mentio1 == 90.85.* ]]; then
252
                                        if [ $MODE == "client" ]; then
253
                                                echo "!!! $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC" | $SENDSOCAT
254
                                        fi
255
                                        echo -e "$RED !!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
256
                                        echo "!!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $BLACKLIST_LOG
257
                                else
258
                                                if [ $MODE == "client" ]; then
259
                                                        echo ">>> $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" | $SENDSOCAT
260
                                                fi                   
261
                                        echo -e "$YELLOW >>> SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
262
                                        echo ">>> $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $DIFF_LOG
263
                                fi
264
                        fi
265
        done
266 1 sacha
    else
267 13 sacha
    echo -e "$GREEN#$count  SITE:$site  $NC"
268 3 sacha
    fi
269 1 sacha
fi
270
271
count=`expr $count + $parallel`
272
273
done
274 14 sacha
##########################################################
275 28 sacha
276 17 sacha
277 14 sacha
278 7 sacha
279 2 sacha
</pre>
280 1 sacha
281 27 sacha
h3. mentio-DNS_ISP_LIST
282 1 sacha
283 27 sacha
<pre>
284
Bouygues 5410 194.158.122.10 194.158.122.15
285
Free 12322 212.27.40.240 212.27.40.241 212.27.40.244 212.27.40.245
286
Orange 3215 80.10.246.1 80.10.246.2 80.10.246.3 80.10.246.5 80.10.246.7 80.10.246.129 80.10.246.130 80.10.246.132 80.10.246.134 80.10.246.136 81.253.149.1 81.253.149.2 81.253.149.6 81.253.149.9 81.253.149.10
287
SFR 15557 109.0.66.10 109.0.66.20
288
</pre>
289 1 sacha
290 27 sacha
h3. mentio-monitor
291 1 sacha
292 27 sacha
<pre>
293
hosts="mentio-HOSTS"
294
hosts_tmp="/tmp/mentio-HOSTS.tmp"
295
rm -rf $hosts
296 1 sacha
297
298 27 sacha
GREEN='\e[32m'
299
RED='\e[31m'
300
NC='\033[0m' # No Color  
301 1 sacha
302 27 sacha
while true; do
303
        timeout 300 tail -n 0 -f MENTIO-DNS_DIFF | cut -d ' ' -f3|sort|uniq> $hosts_tmp
304
        mentiohosts=`cat $hosts_tmp`
305
        updates=`echo $mentiohosts|xargs -n 1`
306
        for i in $updates; do
307
                exists=`grep "$i" $hosts`
308
                if [ -z "$exists" ]; then
309
                        echo "ADD $i"
310
                        echo "$i"  >> $hosts 
311
                        sort -o $hosts $hosts
312
                        name=`echo $i|cut -d "." -f1`
313
                fi
314
        done
315 8 sacha
316 27 sacha
dateus=`date +%Y%m%d-%H%M%S`
317
diffs=`diff --side-by-side --suppress-common-lines $hosts $hosts_tmp`
318
echo "======================================================================"
319
if [ -z "$diffs" ]; then
320
 echo -e "$GREEN $dateus - ALL HOSTS UP: $NC"
321
 echo -e "$GREEN `cat $hosts |xargs |sort` $NC"  
322
else
323
 echo -e "$RED $dateus - MISSING HOST: $NC"
324
 echo -e "$RED $diffs $NC"
325
fi
326
done
327
</pre>
328 1 sacha
329 27 sacha
h2. Test (valide au 14/06/18)
330 1 sacha
331 27 sacha
 dig +short shahamat1.com
332
 90.85.16.52
333 1 sacha
334
h2. Vigies de la neutralité
335
336 18 sacha
https://ooni.torproject.org
337
https://respectmynet.eu
338
339
h2. Cadre légal
340
341
https://www.legifrance.gouv.fr/affichTexte.do?cidTexte=JORFTEXT000000801164&fastPos;=2&fastReqId;=606073666&categorieLien;=cid&oldAction;=rechTexte#LEGIARTI000029756525        
342
le décret https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=LEGITEXT000030315036&dateTexte;=20150305&categorieLien;=cid#LEGITEXT000030315036  
343
https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=JORFTEXT000030195477&dateTexte;=20180619
344
345
h2. Unbound
346
347
<pre>
348
server:
349
 verbosity: 1
350
 interface: 127.0.0.1
351
 do-ip4: yes
352
 do-ip6: no
353
 do-udp: yes
354
 do-tcp: no
355
 access-control: 127.0.0.0/8 allow 
356
 access-control: 0.0.0.0/0 refuse
357 15 sacha
 logfile: /var/log/unbound
358
 hide-identity: yes
359
 hide-version: yes
360
 harden-glue: yes
361
 use-caps-for-id: yes
362
 do-not-query-localhost: yes
363
</pre>
364
365 30 sacha
h2. Lancement
366
367 31 sacha
 tmux new-session -s foo -d "bash mentio-check6 client domain_names.com_sortedac com" \; split-window -h "bash mentio-check6 client domain_names.com_sortedae com" \; split-window -v "bash mentio-check6 client domain_names.com_sortedam com"\; selectp -t 0 \; split-window -v "bash mentio-check6 client domain_names.com_sortedan com"
368 15 sacha
369
h2. Ansible divers
370
371
Copy file:
372
373
 ansible mentio -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
374
375
376
Copy file single host:
377
378
 ansible mentio --limit dam -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
379
380
ansible-playbook /etc/ansible/playbooks/mentio.yml 
381
382
383
<pre>
384
---
385
386
- hosts: mentio 
387
  sudo: no
388
  tasks:
389
    - name: copyfiles 
390
      copy:
391
        src: "{{ item.src }}"
392
        dest: "{{ item.dest }}"
393
      with_items:
394 1 sacha
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-check6',dest: '~/MENTIODNS/' }
395
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-DNS_ISP_LIST',dest: '~/MENTIODNS/' }
396
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.crt',dest: '~/MENTIODNS/' }
397
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.key',dest: '~/MENTIODNS/' }
398
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.pem',dest: '~/MENTIODNS/' }
399
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-server.crt',dest: '~/MENTIODNS/' }
400
401
402
</pre>