Projet

Général

Profil

Mentiodns » Historique » Version 35

sacha, 12/07/2018 12:39

1 1 sacha
h1. Mentiodns
2
3
Validation des DNS à partir d'une liste sur un unbound et un DNS grand FAI et comparaison des résultats ;)
4
5 4 sacha
h2. Noeud actifs
6
7
|_. Nom |_. Bloc |
8 23 sacha
| dam | domain_names.com_sortedaf|
9
| dam | domain_names.com_sortedag|
10
| dam | domain_names.com_sortedah|
11
| dam | domain_names.com_sortedai|
12 24 sacha
| mezzanine | domain_names.com_sortedad |
13
| mezzanine | domain_names.com_sortedao |
14
| mezzanine | domain_names.com_sortedap |
15
| mezzanine | domain_names.com_sortedaq |
16
| 1000i100 | domain_names.com_sortedab |
17
| 1000i100 | domain_names.com_sortedaj |
18
| 1000i100 | domain_names.com_sortedak |
19
| 1000i100 | domain_names.com_sortedal |
20 26 sacha
| sacha | domain_names.org_sortedaa |
21
| sacha | domain_names.org_sortedab |
22
| sacha | domain_names.org_sortedac |
23
| sacha | domain_names.org_sortedad |
24 24 sacha
| tazi | domain_names.com_sortedac |
25
| tazi | domain_names.com_sortedae |
26
| tazi | domain_names.com_sortedam |
27
| tazi | domain_names.com_sortedan |
28 34 sacha
| louisl | domain_names.com_sortedar |
29
| louisl | domain_names.org_sortedae  |
30
| louisl | domain_names.org_sortedaf |
31
| louisl | domain_names.net_sortedaa |
32
| louisl | domain_names.net_sortedab |
33 18 sacha
34 35 sacha
h2. Traités
35
36
domain_names.org_sortedaa 2000000 
37
domain_names.org_sortedab 2000000 
38
39
40 4 sacha
h2. Mentio
41
42 27 sacha
h3. mentio-check6
43
44 19 sacha
Packages: curl dig python socat tmux unbound whois 
45
46 4 sacha
<pre>
47
#-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+--+-#
48
#  MENTIODNS : Check for lying DNS (France)  #
49
#--------------------------------------------#
50 28 sacha
#  Version 1.6.1 date bug
51 16 sacha
#  Version 1.6 - conf file                   # 
52 13 sacha
#  Version 1.5 - test Dig resolving          #
53
#  Version 1.4 - Socat SSL sending results   # 
54
#  Version 1.3 - tld optioN                  #
55 1 sacha
#  Version 1.2 - Round robin on DNS_ISP_LIST #
56 28 sacha
#                For each request            #
57 4 sacha
#  Version 1.1 - Allow resume on basename    #
58
#  Version 1.0 - Parallel process with DIG   #
59
#--------------------------------------------#
60 1 sacha
# (c) Sacha at Aquilenet.fr part of FFDN.org #
61
#--------------------------------------------#
62
63
# This shity script intend to bruteforce the ISP lying DNS Servers to identify which one
64
# is going on Ministry of Interior Blocking page and compare the IP result from your favorite DNS server
65 16 sacha
# Use this script with the following parameters 
66
# $1 MODE: client server local
67
# $2 File source: list of domain names whithout tld
68
# $3 tld: com, org, ...
69
# $4 count number (if none from zero or from count file based on file name)
70
71
72 1 sacha
# If you relanch the script it will check if it has a counter for the given file to resume
73
# Blacklisted sites in $BLACKLIST_LOG file
74 13 sacha
# Diff ip from a domain name are in $DIFF_LOG 
75 1 sacha
76 13 sacha
# 1st launch creating config file
77 1 sacha
78 14 sacha
# Copy generated certificates:
79
# FILENAME=mentio_ssl-server 
80 1 sacha
# openssl genrsa -out $FILENAME.key 1024
81 13 sacha
# openssl req -new -key $FILENAME.key -x509 -days 3653 -out $FILENAME.crt
82 1 sacha
# cat $FILENAME.key $FILENAME.crt >$FILENAME.pem
83 13 sacha
# FILENAME=mentio_ssl-client
84
# ...
85
86 16 sacha
##########################################################
87 1 sacha
HOMEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
88 13 sacha
MENTIOCONF="$HOMEDIR/mentio.conf"
89 17 sacha
# Number of parallel requests thruw dig
90
parallel=10
91 16 sacha
##########################################################
92 1 sacha
93 16 sacha
94
##########################################################
95 13 sacha
### CHECK CONFIG
96
if [ ! -f $MENTIOCONF ]; then
97
echo "=================================================================="
98
echo "MENTIODNS"
99
echo "------------------------------------------------------------------" 
100
echo "1st time configuring"
101
echo -n "IP UNBOUND ? "
102
read DNS_MY
103 1 sacha
104 13 sacha
echo 'HOMEDIR="'$HOMEDIR'"' > $MENTIOCONF
105
echo 'DNS_MY="'$DNS_MY'"' >> $MENTIOCONF
106 1 sacha
IP_PUB=`curl ifconfig.io`
107 13 sacha
ASN=`whois -h whois.cymru.com $IP_PUB |cut -d' ' -f1|sed -n "2p"`
108 1 sacha
DNS_LIST="$HOMEDIR/mentio-DNS_ISP_LIST"
109
DNS_ISP_LIST=`grep $ASN $DNS_LIST|cut -d' ' -f3-`
110
echo 'DNS_ISP_LIST="'$DNS_ISP_LIST'"' >> $MENTIOCONF
111
echo "------------------------------------------------------------------"
112
echo " CONFIGURATION FILE:"
113
echo " please check and relaunch"
114
echo "------------------------------------------------------------------"
115
cat $MENTIOCONF
116
echo "------------------------------------------------------------------"
117
exit 1
118
fi
119 16 sacha
##########################################################
120 1 sacha
121 16 sacha
122
##########################################################
123
### PARAMETERS to execute the script
124
# Mode Log export with socat "client" "server" "local"
125
126
MODE=$1
127
128
if [ $MODE == "server" ]; then
129
socat -v -u openssl-listen:65522,fork,reuseaddr,cert=mentio_ssl-server.pem,cafile=mentio_ssl-client.crt OPEN:$HOMEDIR/MENTIO-DNS_DIFF,creat,append
130
exit 1
131
fi
132
##########################################################
133
134
135
##########################################################
136
### Check if commandline parameters are less than 3
137
138 1 sacha
if [ $# -lt 3 ]; then
139
echo "=================================================================="
140
echo "MENTIODNS"
141
echo "------------------------------------------------------------------"
142 14 sacha
echo "Missing Parameter, please enter:"
143 13 sacha
echo
144 1 sacha
echo "mentio-check client|server|local filename tld (count number)"
145 13 sacha
echo
146
exit 1
147
fi
148 16 sacha
##########################################################
149 13 sacha
150 16 sacha
151
##########################################################
152
### Get parameters
153
154
# From config file
155 1 sacha
source $MENTIOCONF
156 13 sacha
157 16 sacha
# From command line
158 1 sacha
# $2 DNS source file name
159 13 sacha
DNS_SOURCE=$2
160 1 sacha
# $3 TLD name (com, org...)
161 13 sacha
tld=$3
162 1 sacha
# line counter from the dns source file, nothing for auto-resuming
163
COUNT=$4
164
165 16 sacha
##########################################################
166 13 sacha
### SOCAT
167 29 sacha
SERVER="SOMEIP:65522"
168 13 sacha
SENDSOCAT="socat stdio openssl-connect:$SERVER,verify=0,cert=$HOMEDIR/mentio_ssl-client.pem,cafile=$HOMEDIR/mentio_ssl-server.crt"
169 16 sacha
##########################################################
170 13 sacha
### COLORS 
171
RED='\e[31m'
172
GREEN='\e[32m'
173
YELLOW='\e[33m'
174 1 sacha
GRAY='\e[90m'
175
NC='\033[0m' # No Color
176 16 sacha
##########################################################
177
### Various variables
178 13 sacha
DNS_SOURCE_BASENAME=`basename $DNS_SOURCE`
179 1 sacha
DIFF_LOG="$HOMEDIR/DNS_DIFF"
180
BLACKLIST_LOG="$HOMEDIR/DNS_BLACKLISTED"
181 13 sacha
lines=`wc -l $DNS_SOURCE|awk -F " " '{print $1}'`
182 4 sacha
countfile="$HOMEDIR/DNS_Count-$DNS_SOURCE_BASENAME"
183 16 sacha
##########################################################
184
### Dig parameters
185 1 sacha
DIG_FAST="+nodnssec +short +timeout=1 +tries=2"
186
DIG_SLOW="+nodnssec +short +timeout=5 +tries=3 "
187 16 sacha
##########################################################
188 13 sacha
189 16 sacha
190
##########################################################
191
### Generate list for dig: round robin from dns list
192
### Like (@DNS-server domain) x parallel 
193 1 sacha
_check(){
194
i=0
195 13 sacha
url=""
196
while [ $i -lt $parallel ]
197 1 sacha
do
198
n=`expr $count + $i`
199
ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
200
url="$url @$ISP_DNS `awk -v n="${n}" 'NR==n {print;exit}' $DNS_SOURCE`.$tld"
201
i=`expr $i + 1`
202
done
203
}
204 16 sacha
##########################################################
205 1 sacha
206
207 16 sacha
##########################################################
208
### Counter: create one if not existing, use existing instead
209 1 sacha
if [ -z $COUNT ]; then
210 28 sacha
        if [ -f $countfile ]; then
211
        count=`cat $countfile`
212
        else
213
        count=0
214
        echo $count > $countfile
215
        fi
216 13 sacha
else count=$COUNT
217
echo $count > $countfile
218 1 sacha
fi
219 4 sacha
##########################################################
220 16 sacha
221 4 sacha
222 16 sacha
##########################################################
223
### MAIN LOOP
224
225
while [ "$count" != "$lines" ]; do
226 4 sacha
echo $count > $countfile
227
_check
228 28 sacha
dateus=`date +%Y%m%d-%H%M%S`
229 13 sacha
site="$url"
230
echo "-------------------------------------------------------------------------------"
231
echo "#$count $dateus SITE:$site"
232
if nomentio=`dig @$DNS_MY $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio" ]; then
233 28 sacha
        echo -e "$GRAY Unknown zone $site $NC" 
234 13 sacha
fi
235
if mentio=`dig $DIG_SLOW $site|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio" ]; then
236 28 sacha
        echo -e "$GRAY Unknown zone $site $NC"
237 4 sacha
fi
238 1 sacha
239 4 sacha
if [ -n "$nomentio" ] && [ -n "$mentio" ]; then
240 13 sacha
    if [ "$nomentio" != "$mentio" ]; then
241 28 sacha
        for i in $site; do
242
                if nomentio1=`dig $DIG_FAST @$DNS_MY $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$nomentio1" ]; then
243
                        echo -e "$GRAY Unknown zone $i $NC"
244
                fi
245
                ISP_DNS=`echo $DNS_ISP_LIST | xargs -n 1| sort -R | head -n 1`
246
                if mentio1=`dig $DIG_FAST @$ISP_DNS $i|sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4| tr '\r\n' ' '` && [ -z "$mentio1" ]; then
247
                        echo -e "$GRAY Unknown zone $i $NC"
248
                fi
249
                        if [ "$nomentio1" != "$mentio1" ]; then
250
                                if [[ $mentio1 == 90.85.* ]]; then
251
                                        if [ $MODE == "client" ]; then
252
                                                echo "!!! $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC" | $SENDSOCAT
253
                                        fi
254
                                        echo -e "$RED !!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
255
                                        echo "!!! $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $BLACKLIST_LOG
256
                                else
257
                                                if [ $MODE == "client" ]; then
258
                                                        echo ">>> $dateus `hostname` SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" | $SENDSOCAT
259
                                                fi                   
260
                                        echo -e "$YELLOW >>> SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1 $NC"
261
                                        echo ">>> $dateus SITE:$i ISPDNS:$ISP_DNS REAL:$nomentio1 MENTIO:$mentio1" >> $DIFF_LOG
262
                                fi
263
                        fi
264
        done
265 1 sacha
    else
266 13 sacha
    echo -e "$GREEN#$count  SITE:$site  $NC"
267 3 sacha
    fi
268 1 sacha
fi
269
270
count=`expr $count + $parallel`
271
272
done
273 14 sacha
##########################################################
274 28 sacha
275 17 sacha
276 14 sacha
277 7 sacha
278 2 sacha
</pre>
279 1 sacha
280 27 sacha
h3. mentio-DNS_ISP_LIST
281 1 sacha
282 27 sacha
<pre>
283
Bouygues 5410 194.158.122.10 194.158.122.15
284
Free 12322 212.27.40.240 212.27.40.241 212.27.40.244 212.27.40.245
285
Orange 3215 80.10.246.1 80.10.246.2 80.10.246.3 80.10.246.5 80.10.246.7 80.10.246.129 80.10.246.130 80.10.246.132 80.10.246.134 80.10.246.136 81.253.149.1 81.253.149.2 81.253.149.6 81.253.149.9 81.253.149.10
286
SFR 15557 109.0.66.10 109.0.66.20
287
</pre>
288 1 sacha
289 27 sacha
h3. mentio-monitor
290 1 sacha
291 27 sacha
<pre>
292
hosts="mentio-HOSTS"
293
hosts_tmp="/tmp/mentio-HOSTS.tmp"
294
rm -rf $hosts
295 1 sacha
296
297 27 sacha
GREEN='\e[32m'
298
RED='\e[31m'
299
NC='\033[0m' # No Color  
300 1 sacha
301 27 sacha
while true; do
302
        timeout 300 tail -n 0 -f MENTIO-DNS_DIFF | cut -d ' ' -f3|sort|uniq> $hosts_tmp
303
        mentiohosts=`cat $hosts_tmp`
304
        updates=`echo $mentiohosts|xargs -n 1`
305
        for i in $updates; do
306
                exists=`grep "$i" $hosts`
307
                if [ -z "$exists" ]; then
308
                        echo "ADD $i"
309
                        echo "$i"  >> $hosts 
310
                        sort -o $hosts $hosts
311
                        name=`echo $i|cut -d "." -f1`
312
                fi
313
        done
314 8 sacha
315 27 sacha
dateus=`date +%Y%m%d-%H%M%S`
316
diffs=`diff --side-by-side --suppress-common-lines $hosts $hosts_tmp`
317
echo "======================================================================"
318
if [ -z "$diffs" ]; then
319
 echo -e "$GREEN $dateus - ALL HOSTS UP: $NC"
320
 echo -e "$GREEN `cat $hosts |xargs |sort` $NC"  
321
else
322
 echo -e "$RED $dateus - MISSING HOST: $NC"
323
 echo -e "$RED $diffs $NC"
324
fi
325
done
326
</pre>
327 1 sacha
328 27 sacha
h2. Test (valide au 14/06/18)
329 1 sacha
330 27 sacha
 dig +short shahamat1.com
331
 90.85.16.52
332 1 sacha
333
h2. Vigies de la neutralité
334
335 18 sacha
https://ooni.torproject.org
336
https://respectmynet.eu
337
338
h2. Cadre légal
339
340
https://www.legifrance.gouv.fr/affichTexte.do?cidTexte=JORFTEXT000000801164&fastPos;=2&fastReqId;=606073666&categorieLien;=cid&oldAction;=rechTexte#LEGIARTI000029756525        
341
le décret https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=LEGITEXT000030315036&dateTexte;=20150305&categorieLien;=cid#LEGITEXT000030315036  
342
https://www.legifrance.gouv.fr/affichTexte.do;jsessionid=FE6BFDED672BF1E2EFC5CA70705CF26E.tplgfr21s_3?cidTexte=JORFTEXT000030195477&dateTexte;=20180619
343
344
h2. Unbound
345
346
<pre>
347
server:
348
 verbosity: 1
349
 interface: 127.0.0.1
350
 do-ip4: yes
351
 do-ip6: no
352
 do-udp: yes
353
 do-tcp: no
354
 access-control: 127.0.0.0/8 allow 
355
 access-control: 0.0.0.0/0 refuse
356 15 sacha
 logfile: /var/log/unbound
357
 hide-identity: yes
358
 hide-version: yes
359
 harden-glue: yes
360
 use-caps-for-id: yes
361
 do-not-query-localhost: yes
362
</pre>
363
364 30 sacha
h2. Lancement
365
366 31 sacha
 tmux new-session -s foo -d "bash mentio-check6 client domain_names.com_sortedac com" \; split-window -h "bash mentio-check6 client domain_names.com_sortedae com" \; split-window -v "bash mentio-check6 client domain_names.com_sortedam com"\; selectp -t 0 \; split-window -v "bash mentio-check6 client domain_names.com_sortedan com"
367 15 sacha
368
h2. Ansible divers
369
370
Copy file:
371
372
 ansible mentio -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
373
374
375
Copy file single host:
376
377
 ansible mentio --limit dam -m copy -a "src=mentio-check6 dest=~/MENTIODNS/"
378
379
ansible-playbook /etc/ansible/playbooks/mentio.yml 
380
381
382
<pre>
383
---
384
385
- hosts: mentio 
386
  sudo: no
387
  tasks:
388
    - name: copyfiles 
389
      copy:
390
        src: "{{ item.src }}"
391
        dest: "{{ item.dest }}"
392
      with_items:
393 1 sacha
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-check6',dest: '~/MENTIODNS/' }
394
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio-DNS_ISP_LIST',dest: '~/MENTIODNS/' }
395
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.crt',dest: '~/MENTIODNS/' }
396
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.key',dest: '~/MENTIODNS/' }
397
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-client.pem',dest: '~/MENTIODNS/' }
398
        - { src: '/home/sacha/0nmyway/00_Aquilenet/FFDN/mentiodns.fr/mentio_ssl-server.crt',dest: '~/MENTIODNS/' }
399
400
401
</pre>