DNS is fundamental to how the
Found at: gopher.blog.benjojo.co.uk:70/ISPs-sharing-DNS-query-data
The ISPs sharing your DNS query data
===
DNS is fundamental to how the web works, and for most of
the population it's completely transparent. Everything on the web
is accessed by a DNS name.
Since DNS is an old protocol (November 1987 in fact) the
protocol was built in a time where encryption was not viewed as a
necessary thing. This is not to be confused with DNSSEC, whose
purpose is to validate that a DNS response has not been tampered
with.
Because queries are not encrypted in DNS, they are visible
to the network devices between you and the resolver, and even
if they were, in almost all cases people use external
resolvers either locally in their network (normally a DNS query cache
that passes on misses to a ISP DNS resolver) or an external
one (see 8.8.8.8 / 9.9.9.9 / 1.1.1.1 and other catchy IP
addresses)
A lot of "threat intelligence" relies on DNS data for
research, so security teams inside of companies often log DNS queries
so they can track down incidents later on.
However to make this data meaningful, researchers need data
from real systems in the wild. This is where they turn to
larger companies who have data feeds from large resolvers silently
logging queries that pass through them.
DNS Diagram
This means that downstream customers' data is sent away to
3rd party sources, who often charge to have researchers access
the data.
So what ISP's are doing this? How can we find that out?
You just ask every open DNS resolver a query that is tied
to the ISP itself! After all, the IPv4 internet is only 3.5~
million routable addresses!
So over ~18 hours my server send 50,000 DNS packets per
second to random internet hosts.
RRDTool graph of bandwidth
To make this setup work, I bought a new domain name for
this experiment and for every single ASN) generated a specific
DNS query for that ISP. On the sending side I discarded any
responses and only listened to queries that made it back to my
domains DNS nameservers:
DNS Diagram 2
The idea is that a lot of these resolvers I am hitting
are actually just caches that upstream to their local ISP
resolver. From the results of the inbound query I can see what
ISP’s use what ISPs to send the lookups to my nameserver, using
the results of the inbound query. A query sent to a home ISP
that results in a nameserver query arriving from Google would
suggest that 8.8.8.8 is in use, a query arriving from cloudflare
would suggest 1.1.1.1, or a query arriving from the ISP itself
would suggest it's the ISPs local infrastructure looking up the
query.
In addition, we can look up on the open platforms to see
what subdomains were harvested:
A DNS harvester lookup
There doesn't seem to be a direct link between these ASN's
and their resolvers (other than google, who is unlikely to be
only selling a very small % of their queries)
There is another interesting phenomena. Even when scanning
was finished, I was still getting queries as if I was still
sending packets to some ISPs. I've come to call these "zombie
queries"
Here is an example:
I sent 2 million queries to Telenet in Belgium, and got
416 queries to my name servers back. However after scanning I
got 4 queries from a South African ISP:
```
2018-06-20 05:27:15 IP 197.96.129.180.51587 > 45.76.133.249.53:
32906 [1au] A? a-6848.4uqu.party. (46)
2018-06-20 05:28:51 IP 197.96.129.180.39401 > 45.76.133.249.53:
20872 [1au] ANY? a-6848.4uqu.party. (46)
2018-06-20 05:28:51 IP 197.96.129.180.35844 > 45.76.133.249.53:
2062 [1au] ANY? a-6848.4uqu.party. (46)
2018-06-20 05:28:52 IP 197.96.129.180.60279 > 45.76.133.249.53:
14686 [1au] ANY? a-6848.4uqu.party. (46)
```
This is especially curious, since the last 3 of those
queries are for a different DNS type. So something has to have
processed the query sent to Telenet in Belgium and decided to poke
at my subdomain more.
This does not 100% mean that Telenet has sold data to a
3rd party that then re-queried me in South Africa, it could
have been a customer with a config that pointed to another
upstream DNS provider that sold it.
You can explore below the data I have gathered, You can
enter ISP names or the AS number:
Enter a AS number in theformat of "ASXXX" or just freetype a nameSearch:type="text" id="blogsearch">ISP Name:NonePlease enter a queryid="resolvers">Resolvers:
AS Persentage Raw NumberZombie resolvers:
style="width:100%">AS Raw Number
(function(C,B){"function"===typeof
define&&define.amd?define([],B):"object"===typeof
module&&module.exports?module.exports=B():C.fuzzysort=B()})(this,function(){function C(c){var f={single:function(a,d,b){if(!a)return
null;D(a)||(a=f.getPreparedSearch(a));if(!d)return null;D(d)||(d=f.getPrepared(d));return((b&&void
0!==b.allowTypo?b.allowTypo:c&&void
0!==c.allowTypo?c.allowTypo:1)?f.algorithm:f.algorithmNoTypo)(a,d,a[0])},go:function(a,d,b){if(!a)return F;a=f.prepareSearch(a);var E=a[0],n=b&&b.threshold||
c&&c.threshold||-9007199254740991,m=b&&b.limit||c&&c.limit||9007199254740991,g=(b&&void 0!==b.allowTypo?b.allowTypo:c&&void
0!==c.allowTypo?c.allowTypo:1)?f.algorithm:f.algorithmNoTypo,e=0,l=0,k=d.length;if(b&&b.keys){var h=b.scoreFn||K;b=b.keys;var v=b.length;for(--k;0<=k;--k){for(var
p=d[k],u=Array(v),t=v-1;0<=t;--t){var
z=b[t],w=G(p,z);w?(D(w)||(w=f.getPrepared(w)),u[t]=g(a,w,E)):u[t]=null}u.obj=p;z=h(u);null===z||zA.peek().score&&A.replaceTop(u)))}}else if(b&&
b.key)for(z=b.key,--k;0<=k;--k){if(p=d[k],w=G(p,z))D(w)||(w=f.getPrepared(w)),h=g(a,w,E),null===h||h.scoreA.peek().score&&A.replaceTop(h)))}else
for(--k;0<=k;--k)if(w=d[k])D(w)||(w=f.getPrepared(w)),h=g(a,w,E),null===h||h.scoreA.peek().score&&A.replaceTop(h)));if(0===e)return F;a=Array(e);for(k=e-1;0<=k;--k)a[k]=A.poll();
a.total=e+l;return a},goAsync:function(a,d,b){var E=!1,n=new
Promise(function(n,g){function e(){if(E)return g("canceled");var
c=Date.now();if(b&&b.keys)for(var q=b.scoreFn||K,v=b.keys,A=v.length;0<=h;--h){for(var
r=d[h],y=Array(A),B=A-1;0<=B;--B){var
C=v[B],x=G(r,C);x?(D(x)||(x=f.getPrepared(x)),y[B]=u(a,x,l)):y[B]=null}y.obj=r;r=q(y);if(null!==r&&!(rk.peek().score&&k.replaceTop(y)),0===h%1E3&&10<=Date.now()-c)){H?setImmediate(e):setTimeout(e);return}}else if(b&&
b.key)for(C=b.key;0<=h;--h){if(r=d[h],x=G(r,C))if(D(x)||(x=f.getPrepared(x)),q=u(a,x,l),null!==q&&!(q.scorek.peek().score&&k.replaceTop(q)),0===h%1E3&&10<=Date.now()-c)){H?setImmediate(e):setTimeout(e);return}}else
for(;0<=h;--h)if(x=d[h])if(D(x)||(x=f.getPrepared(x)),q=u(a,x,l),null!==q&&!(q.scorek.peek().score&&k.replaceTop(q)),
0===h%1E3&&10<=Date.now()-c)){H?setImmediate(e):setTimeout(e);return}if(0===t)return
n(F);c=Array(t);for(q=t-1;0<=q;--q)c[q]=k.poll();c.total=t+z;n(c)}if(!a)return n(F);a=f.prepareSearch(a);var
l=a[0],k=L(),h=d.length-1,m=b&&b.threshold||c&&c.threshold||-9007199254740991,p=b&&b.limit||c&&c.limit||9007199254740991,u=(b&&void 0!==b.allowTypo?b.allowTypo:c&&void
0!==c.allowTypo?c.allowTypo:1)?f.algorithm:f.algorithmNoTypo,t=0,z=0;H?setImmediate(e):e()});n.cancel=function(){E=!0};return n},highlight:function(a,
d,b){if(null===a)return null;void 0===d&&(d="");void
0===b&&(b="");var f="",n=0,c=!1,g=a.target,e=g.length;a=a.indexes;for(var
l=0;l k=g[l];if(a[n]===l){if(++n,c||(c=!0,f+=d),n===a.length){f+=k+b+g.substr(l+1);break}}else c&&(c=!1,f+=b);f+=k}returnf},prepare:function(a){if(a)return{target:a,_targetLowerCodes:f.prepareLowerCodes(a),_nextBeginningIndexes:null,score:null,indexes:null,obj:null}},prepareSlow:function(a){if(a)return{target:a,_targetLowerCodes:f.prepareLowerCodes(a),_nextBeginningIndexes:f.prepareNextBeginningIndexes(a),score:null,indexes:null,obj:null}},prepareSearch:function(a){if(a)returnf.prepareLowerCodes(a)},getPrepared:function(a){if(999 d;d=f.prepare(a);I.set(a,d);return d},getPreparedSearch:function(a){if(999 f.prepareSearch(a);var d=J.get(a);if(void 0!==d)returnd;d=f.prepareSearch(a);J.set(a,d);return d},algorithm:function(a,d,b){for(varc=d._targetLowerCodes,n=a.length,m=c.length,g=0,e=0,l=0,k=0;;){var h=b===c[e];if(h){r[k++]=e;++g;if(g===n)break;b=a[0===l?g:l===g?g+1:l===g-1?g-1:g]}++e;if(e>=m)for(;;){if(1>=g)return null;if(0===l){--g;e=a[g];if(b===e)continue;l=g}else{if(1===l)returnnull;--l;g=l;b=a[g+1];e=a[g];if(b===e)continue}k=g;e=r[k-1]+1;break}}b=g=0;varv=!1,p=0,u=d._nextBeginningIndexes;null===u&&(u=d._nextBeginningIndexes=f.prepareNextBeginningIndexes(d.target));vart=e=0===r[0]?0:u[r[0]-1];if(e!==m)for(;;)if(e>=m)if(0>=g){++b;if(b>n-2)break;a[b]!==a[b+1]&&(e=t)}else--g,e=y[--p],e=u[e];else if(h=a[0===b?g:b===g?g+1:b===g-1?g-1:g]===c[e]){y[p++]=e;++g;if(g===n){v=!0;break}++e}elsee=u[e];v?(a=y,c=p):(a=r,c=k);g=0;k=-1;for(p=0;p c=d._targetLowerCodes,n=a.length,m=c.length,g=0,e=0,l=0;;){vark=b===c[e];if(k){r[l++]=e;++g;if(g===n)break;b=a[g]}++e;if(e>=m)return null}g=0;b=!1;var h=0,v=d._nextBeginningIndexes;null===v&&(v=d._nextBeginningIndexes=f.prepareNextBeginningIndexes(d.target));e=0===r[0]?0:v[r[0]-1];if(e!==m)for(;;)if(e>=m){if(0>=g)break;--g;e=y[--h];e=v[e]}else if(k=a[g]===c[e]){y[h++]=e;++g;if(g===n){b=!0;break}++e}elsee=v[e];b?(a=y,l=h):a=r;c=0;g=-1;for(h=0;h d=a.length,b=[];a=a.toLowerCase();for(var f=0;f prepareBeginningIndexes:function(a){for(vard=a.length,b=[],f=0,c=!1,m=!1,g=0;g e=a.charCodeAt(g),l=65<=e&&90>=e;e=l||97<=e&&122>=e||48<=e&&57>=e;var k=l&&!c||!m||!e;c=l;m=e;k&&(b[f++]=g)}returnb},prepareNextBeginningIndexes:function(a){var d=a.length;a=f.prepareBeginningIndexes(a);for(varb=[],c=a[0],n=0,m=0;mm?b[m]=c:(c=a[++n],b[m]=void 0===c?d:c);return b},cleanup:B,"new":C};return f}function B(){I.clear();J.clear();r=[];y=[]}function K(c){for(var f=-9007199254740991,a=c.length-1;0<=a;--a){vard=c[a];null!==d&&(d=d.score,d>f&&(f=d))}return-9007199254740991===f?null:f}function G(c,f){var a=c[f];if(void 0!==a)returna;a=f;Array.isArray(f)||(a=f.split("."));for(var d=a.length,b=-1;c&&++b D(c){return"object"===typeof c}var H="undefined"!==typeof require&&"undefined"===typeofwindow,I=new Map,J=new Map,F=[];F.total=0;var r=[],y=[],L=function(){functionc(){for(var b=0,d=f[b],c=1;cm=c+1;b=c;m>1]=f[b];c=1+(b<<1)}for(c=b-1>>1;0>1)f[b]=f[c];f[b]=d}var f=[],a=0,d={};return d.add=function(b){var c=a;f[a++]=b;for(vard=c-1>>1;0>1)f[c]=f[d];f[c]=b},d.poll=function(){if(0!==a){var b=f[0];return f[0]=f[--a],c(),b}},d.peek=function(b){if(0!==a)return f[0]},d.replaceTop=function(a){f[0]=a;c()},d},A=L();return C()});var request = new XMLHttpRequest();request.open('GET', '/asset/gbz8jDkewV', true);var datadownloaded = false;var globaldata = {};var asnames = [];var asmap = [];var namestoas = [];notes = document.getElementById("notes");notes.textContent = "Data is downloading, Pleasewait...";request.onload = function() {if (request.status >= 200 && request.status < 400) {// Success!var data = JSON.parse(request.responseText);globaldata = data;datadownloaded = true;Object.keys(globaldata.Name).forEach(function(key) {// console.log(key, globaldata.Name[key]);asnames.push(globaldata.Name[key]);asmap[key] = globaldata.Name[key];namestoas[globaldata.Name[key]] = key;notes = document.getElementById("notes");notes.textContent = "Data Downloaded! Ready toquery";});} else {// We reached our target server, but it returned anerrornotes = document.getElementById("notes");notes.textContent = "OOPSIE WOOPSIE!! Uwu We made afucky wucky!! A wittle fucko boingo! (Sorry) You won't be ableto query data, try reloading the page maybe?";}};request.onerror = function() {// There was a connection error of some sort};isAppleSafari = function() {var is_safari =navigator.userAgent.toLowerCase().indexOf('safari/') > -1;var is_apple = navigator.userAgent.toLowerCase().indexOf('apple')> -1;var is_chrome =navigator.userAgent.toLowerCase().indexOf('chrome/') > -1;return is_safari && !is_chrome && is_apple}if (!isAppleSafari()) {request.send();} else {notes = document.getElementById("notes");notes.textContent = "Safari is a broken browser thatcannot render this javascript without crashing. Use a better mobileOS.";}function render(asn) {ispname = document.getElementById("ispname");ispname.textContent = "ISP Name = " +asmap[parseInt(asn)];realasn = parseInt(asn);// okay now to lookup dataquerydata = globaldata.AS[asn];notes = document.getElementById("notes");notes.textContent = "";if (querydata == undefined) {notes.textContent = "I have no information onthis ISP";ispname.textContent = "";return}if (querydata.PT != 0) {notes.textContent = "This ASN has resolversthat report to harvesting services"}resolvers = document.getElementById("resolvers");mega = "Resolvers:
"
style=\"width:100%\">";mega += "" AS % Query CountObject.keys(querydata.QASN).forEach(function(key) {mega += "" " +asmap[parseInt(key)] + ""+Math.floor((querydata.QASN[key] / querydata.C)*100) +" %"+querydata.QASN[key]+" })mega += "resolvers.innerHTML = megazombies = document.getElementById("zombieresolvers");mega = "Zombie Resolers:
"
style=\"width:100%\">";mega += "" AS Query Count count = 0;Object.keys(querydata.PSQAS).forEach(function(key) {count++;mega += "" " +asmap[parseInt(key)] + ""+querydata.PSQAS[key]+" })mega += "if(count == 0) {zombies.innerHTML = ""} else {zombies.innerHTML = mega}console.log(globaldata.AS[asn])};ispbox = document.getElementById("blogsearch");ispbox.addEventListener("keyup", function() {if(!datadownloaded) {return}if(ispbox.value.lastIndexOf("AS", 0) === 0) {// ASN logicASNumber = ispbox.value.substring(2,999);render(ASNumber)return}// Fuzzy name searchfuzzyresults = fuzzysort.go(ispbox.value, asnames);if(fuzzyresults.length == 0) {ispname = document.getElementById("ispname");ispname.textContent = "ISP Name = ";notes = document.getElementById("notes");notes.textContent = "No ISP found with thatname";return}asn = namestoas[fuzzyresults[0].target]render(asn)});You can find the tools I wrote for this over at:https://github.com/benjojo/dns-spiesIf you enjoyed this you can follow my Twitter or RSS tokeep up with the other silly (or sometimes sensible) things Iwill do!Until next time!