Skip to content

Commit 3c949fb

Browse files
committed
Integration of anura, mastercidr, and geopatch
1 parent d59ca84 commit 3c949fb

File tree

10 files changed

+282
-8
lines changed

10 files changed

+282
-8
lines changed

Campaigns/docker.json

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"watchlist": "$FRAUDWATCHLIST"
1111
},
1212
"app" : {
13+
"geopatch": "$GEOPATCH",
1314
"indexPage": "$INDEXPAGE",
1415
"concurrency" : "3",
1516
"threads": "2000",

Campaigns/payday.json

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"watchlist": "$FRAUDWATCHLIST"
1111
},
1212
"app" : {
13+
"geopatch": "$GEOPATCH",
1314
"concurrency" : "1",
1415
"NOdeadmanswitch":"accountingsystem",
1516
"threads": "128",

Docker.bidder

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ COPY web/* /web/
4141
COPY log4j.properties /
4242
COPY SampleBids /SampleBids
4343

44-
COPY database.json /
44+
#COPY database.json /
4545

4646
COPY Campaigns/docker.json Campaigns/payday.json
4747

SampleBids/nogeo.txt

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"cur": [
3+
"USD"
4+
],
5+
"id": "VgpV7SMBP70zFzseiG8TLQ",
6+
"imp": [{
7+
"bidfloorcur": "USD",
8+
"tagid": "1234",
9+
"banner": {
10+
"id": "1",
11+
"h": 50,
12+
"w": 320
13+
},
14+
"id": "1"
15+
}],
16+
"site": {
17+
"publisher": {
18+
"name": "Sample Publisher",
19+
"id": "3456"
20+
},
21+
"ref": "http://www.sample.com/",
22+
"cat": [
23+
"IAB25-3"
24+
],
25+
"domain": "sample.com",
26+
"name": "Sample Site",
27+
"id": "6789"
28+
},
29+
"device": {
30+
"connectiontype": 1,
31+
"carrier": "Unknown",
32+
"js": 1,
33+
"os": "Windows",
34+
"devicetype": 2,
35+
"ip": "12.23.34.45",
36+
"ua": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET4.0C; .NET4.0E)"
37+
},
38+
"user": {
39+
"id": "VgpV7eIBBSs1iQnp6-mY-g"
40+
},
41+
"test": 0,
42+
"at": 2,
43+
"tmax": 100
44+
}

src/com/jacamars/dsp/rtb/bidder/Controller.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -958,7 +958,8 @@ public void sendStats(Map m) {
958958
*/
959959

960960
public boolean sendRequest(BidRequest br, boolean override) throws Exception {
961-
if (br.notABidRequest())
961+
962+
if (br.notABidRequest())
962963
return false;
963964

964965
if (!override) {
@@ -986,7 +987,6 @@ public boolean sendRequest(BidRequest br, boolean override) throws Exception {
986987
}
987988
original.put("logtype", "requests");
988989
requestQueue.addString(original.toString());
989-
990990
}
991991

992992
return true;

src/com/jacamars/dsp/rtb/bidder/RTBServer.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -1155,7 +1155,6 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques
11551155
Controller.getInstance().sendRequest(br, false);
11561156
return;
11571157
}
1158-
11591158
// Some exchanges like Appnexus send other endpoints, so
11601159
// they are handled here.
11611160
if (br.notABidRequest()) {
@@ -1167,8 +1166,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques
11671166
RTBServer.request--;
11681167
return;
11691168
}
1170-
1171-
if (Configuration.getInstance().getCampaignsList().size() == 0) {
1169+
if (Configuration.getInstance().getCampaignsList().size() == 0) {
11721170
logger.debug("No campaigns loaded");
11731171
json = br.returnNoBid("No campaigns loaded");
11741172
code = RTBServer.NOBID_CODE;
@@ -1194,6 +1192,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques
11941192
json = br.returnNoBid("No matching campaign");
11951193
code = RTBServer.NOBID_CODE;
11961194
RTBServer.nobid++;
1195+
11971196
Controller.getInstance().sendRequest(br, false);
11981197
Controller.getInstance().sendNobid(new NobidResponse(br.id, br.getExchange()));
11991198
} else {

src/com/jacamars/dsp/rtb/common/Configuration.java

+17-2
Original file line numberDiff line numberDiff line change
@@ -480,12 +480,14 @@ public void initialize(String path, String shard, int port, int sslPort, String
480480
if (x != null) {
481481
if (x instanceof NavMap) {
482482
masterCidr = (NavMap) x;
483+
logger.info("*** Master Blacklist is set to: {}",x);
483484
} else {
484-
logger.error("Master CIDR '@MASTERCIDR' is the wrong classtype {}", x.getClass().getName());
485+
logger.error("*** Master CIDR '@MASTERCIDR' is the wrong classtype {}", x.getClass().getName());
485486
logger.error("*** Master CIDR blocking is disabled ***");
486487
}
487488
}
488-
}
489+
} else
490+
logger.info("*** Master Blacklist is not set");
489491
/**
490492
* SSL
491493
*/
@@ -579,6 +581,16 @@ public void initialize(String path, String shard, int port, int sslPort, String
579581
* Deal with the app object
580582
*/
581583
m = (Map) m.get("app");
584+
585+
if (m.get("geopatch") != null) {
586+
String fileName = (String)m.get("geopatch");
587+
if (!fileName.equals("")) {
588+
GeoPatch.getInstance(fileName);
589+
logger.info("*** GEOPATCH DB set to: {} ",fileName);
590+
} else
591+
logger.info("*** GEOPATCH DB IS NOT SET");
592+
} else
593+
logger.info("*** GEOPATCH DB IS NOT SET");
582594

583595
if (m.get("indexPage") != null)
584596
indexPage = (String) m.get("indexPage");
@@ -882,6 +894,9 @@ public static String substitute(String address) throws Exception {
882894
if (address == null)
883895
return address;
884896

897+
while (address.contains("$GEOPATCH"))
898+
address = GetEnvironmentVariable(address, "$GEOPATCH", "");
899+
885900
while (address.contains("$MASTERCIDR"))
886901
address = GetEnvironmentVariable(address, "$MASTERCIDR", "");
887902

src/com/jacamars/dsp/rtb/pojo/BidRequest.java

+2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.jacamars.dsp.rtb.fraud.FraudLog;
2424
import com.jacamars.dsp.rtb.geo.Solution;
2525
import com.jacamars.dsp.rtb.tools.AmalgamatedKey;
26+
import com.jacamars.dsp.rtb.tools.GeoPatch;
2627
import com.jacamars.dsp.rtb.tools.HexDump;
2728
import com.jacamars.dsp.rtb.tools.IsoTwo2Iso3;
2829
import com.fasterxml.jackson.databind.JsonNode;
@@ -473,6 +474,7 @@ protected void setup() throws Exception {
473474
}
474475

475476
makeSynthKey();
477+
GeoPatch.getInstance().patch(rootNode.get("device"));
476478

477479
// ///////////////
478480
if (siteDomain != null && blackList != null) {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
package com.jacamars.dsp.rtb.tools;
2+
import java.io.File;
3+
import java.net.InetAddress;
4+
5+
import com.fasterxml.jackson.databind.JsonNode;
6+
import com.fasterxml.jackson.databind.node.DoubleNode;
7+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
8+
import com.fasterxml.jackson.databind.node.MissingNode;
9+
import com.fasterxml.jackson.databind.node.ObjectNode;
10+
import com.fasterxml.jackson.databind.node.TextNode;
11+
import com.google.openrtb.OpenRtb.BidRequest;
12+
import com.jacamars.dsp.rtb.blocks.LookingGlass;
13+
import com.jacamars.dsp.rtb.exchanges.Atomx;
14+
import com.jacamars.dsp.rtb.tools.IsoTwo2Iso3;
15+
import com.maxmind.geoip2.DatabaseReader;
16+
import com.maxmind.geoip2.model.CityResponse;
17+
import com.maxmind.geoip2.record.City;
18+
import com.maxmind.geoip2.record.Country;
19+
import com.maxmind.geoip2.record.Postal;
20+
import com.maxmind.geoip2.record.Subdivision;
21+
import com.maxmind.geoip2.record.Location;
22+
23+
/**
24+
* A singleton that adds geo information to a device, if IP is available.
25+
* @author Ben M. Faul
26+
*
27+
*/
28+
public enum GeoPatch {
29+
GEOPATCH;
30+
static final JsonNodeFactory factory = JsonNodeFactory.instance;
31+
static IsoTwo2Iso3 isoMap;
32+
static DatabaseReader reader = null;
33+
34+
public static void main(String[] args) throws Exception {
35+
new IsoTwo2Iso3("@ISO2-3", "data/adxgeo.csv");
36+
GeoPatch.getInstance("/home/ben/GeoLite2-City.mmdb");
37+
Atomx br = new Atomx("SampleBids/nogeo.txt");
38+
}
39+
40+
public static GeoPatch getInstance(String fileName) throws Exception {
41+
reader = new DatabaseReader.Builder(new File(fileName)).build();
42+
isoMap = (IsoTwo2Iso3) LookingGlass.symbols.get("@ISO2-3");
43+
InetAddress ipAddress = InetAddress.getByName("47.180.117.78");
44+
reader.city(ipAddress);
45+
return GEOPATCH;
46+
}
47+
48+
public static GeoPatch getInstance() {
49+
return GEOPATCH;
50+
}
51+
52+
public void patch(JsonNode idev) throws Exception {
53+
54+
//long timer = System.currentTimeMillis();
55+
/**
56+
* No patch if not initialized
57+
*/
58+
if (reader == null || idev == null || idev instanceof MissingNode)
59+
return;
60+
61+
ObjectNode device = (ObjectNode)idev;
62+
/**
63+
* Make sure iso2 to 3 converter is available.
64+
*/
65+
if (isoMap == null)
66+
isoMap = (IsoTwo2Iso3) LookingGlass.symbols.get("@ISO2-3");
67+
68+
String ip = device.get("ip").asText("");
69+
if (ip.equals(""))
70+
return;
71+
72+
InetAddress ipAddress = InetAddress.getByName(ip);
73+
CityResponse response = null;
74+
City city = null;
75+
Country country = null;
76+
Subdivision subdivision = null;
77+
Postal postal = null;
78+
Location location = null;
79+
80+
/**
81+
* Make a geo node if necessary
82+
*/
83+
ObjectNode geo = (ObjectNode)device.get("geo");
84+
if (geo == null) {
85+
geo = factory.objectNode();
86+
device.set("geo", geo);
87+
response = reader.city(ipAddress);
88+
city = response.getCity();
89+
country = response.getCountry();
90+
subdivision = response.getMostSpecificSubdivision();
91+
postal = response.getPostal();
92+
location = response.getLocation();
93+
}
94+
95+
/**
96+
* Add country, convert iso2 to iso3
97+
*/
98+
JsonNode x = null;
99+
if ((x = geo.get("country"))==null) {
100+
if (x == null) {
101+
if (response == null)
102+
response = reader.city(ipAddress);
103+
if (city == null)
104+
city = response.getCity();
105+
if (country == null)
106+
country = response.getCountry();
107+
if (subdivision == null)
108+
subdivision = response.getMostSpecificSubdivision();
109+
if (postal == null)
110+
postal = response.getPostal();
111+
if (location == null)
112+
location = response.getLocation();
113+
}
114+
String iso = country.getIsoCode();
115+
if (iso.length()==2 && isoMap != null) {
116+
iso = isoMap.query(iso);
117+
}
118+
geo.set("country", new TextNode(iso));
119+
}
120+
121+
/**
122+
* Add city
123+
*/
124+
x = null;
125+
if ((x = geo.get("city"))==null) {
126+
if (x == null) {
127+
if (response == null)
128+
response = reader.city(ipAddress);
129+
if (city == null)
130+
city = response.getCity();
131+
if (country == null)
132+
country = response.getCountry();
133+
if (subdivision == null)
134+
subdivision = response.getMostSpecificSubdivision();
135+
if (postal == null)
136+
postal = response.getPostal();
137+
if (location == null)
138+
location = response.getLocation();
139+
}
140+
geo.set("city", new TextNode(city.getName()));
141+
}
142+
143+
/**
144+
* Add state/region
145+
*/
146+
x = null;;
147+
if ((x = geo.get("region"))==null) {
148+
if (x == null) {
149+
if (response == null)
150+
response = reader.city(ipAddress);
151+
if (city == null)
152+
city = response.getCity();
153+
if (country == null)
154+
country = response.getCountry();
155+
if (subdivision == null)
156+
subdivision = response.getMostSpecificSubdivision();
157+
if (postal == null)
158+
postal = response.getPostal();
159+
if (location == null)
160+
location = response.getLocation();
161+
}
162+
geo.set("state",new TextNode(subdivision.getIsoCode()));
163+
}
164+
x = null;;
165+
if ((x = geo.get("zipcode"))==null) {
166+
if (x == null) {
167+
if (response == null)
168+
response = reader.city(ipAddress);
169+
if (city == null)
170+
city = response.getCity();
171+
if (country == null)
172+
country = response.getCountry();
173+
if (subdivision == null)
174+
subdivision = response.getMostSpecificSubdivision();
175+
if (postal == null)
176+
postal = response.getPostal();
177+
if (location == null)
178+
location = response.getLocation();
179+
}
180+
geo.set("zip", new TextNode(postal.getCode()));
181+
}
182+
183+
/**
184+
* Add latitude and longitude
185+
*/
186+
x = null;;
187+
if ((x = geo.get("lat"))==null) {
188+
if (x == null) {
189+
if (response == null)
190+
response = reader.city(ipAddress);
191+
if (city == null)
192+
city = response.getCity();
193+
if (country == null)
194+
country = response.getCountry();
195+
if (subdivision == null)
196+
subdivision = response.getMostSpecificSubdivision();
197+
if (postal == null)
198+
postal = response.getPostal();
199+
if (location == null)
200+
location = response.getLocation();
201+
}
202+
geo.set("lat", new DoubleNode(location.getLatitude()));
203+
geo.set("lon", new DoubleNode(location.getLatitude()));
204+
}
205+
206+
//timer = System.currentTimeMillis() - timer;
207+
//System.out.println(timer);
208+
}
209+
210+
}

web/admin.html

+2
Original file line numberDiff line numberDiff line change
@@ -1113,6 +1113,8 @@ <h3 class="panel-title">Templates</h3>
11131113

11141114
function fillInInits(data) {
11151115
var inits = data.initials;
1116+
if (typeof inits === "undefined")
1117+
return;
11161118
initLoadsDyno.clear();
11171119
for (var i=0; i<inits.length;i++) {
11181120
var camp = inits[i];

0 commit comments

Comments
 (0)