Skip to content

Commit 8b4012a

Browse files
irjudsonirjudson
committed
Added files.
1 parent 0b06603 commit 8b4012a

File tree

6 files changed

+250
-0
lines changed

6 files changed

+250
-0
lines changed

‎ANONYMIZE.java

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package jaglion;
2+
3+
import java.io.IOException;
4+
import java.util.UUID;
5+
6+
import org.apache.commons.codec.digest.DigestUtils;
7+
8+
import org.apache.pig.EvalFunc;
9+
import org.apache.pig.data.Tuple;
10+
11+
import org.apache.hadoop.conf.Configuration;
12+
import org.apache.hadoop.hbase.HBaseConfiguration;
13+
import org.apache.hadoop.hbase.HTableDescriptor;
14+
import org.apache.hadoop.hbase.HColumnDescriptor;
15+
import org.apache.hadoop.hbase.MasterNotRunningException;
16+
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
17+
import org.apache.hadoop.hbase.client.HBaseAdmin;
18+
import org.apache.hadoop.hbase.client.HTable;
19+
import org.apache.hadoop.hbase.client.Put;
20+
import org.apache.hadoop.hbase.util.Bytes;
21+
22+
public class ANONYMIZE extends EvalFunc<String>
23+
{
24+
public Configuration config;
25+
public String fmap = "FowardMap";
26+
public String rmap = "ReversMap";
27+
28+
public String exec(Tuple input) throws IOException {
29+
30+
// As long as we have a reasonable inputs
31+
if (input.size() != 2 || input.get(0) == null || input.get(1) == null)
32+
return null;
33+
34+
// Connect to HBASE
35+
config = HBaseConfiguration.create();
36+
HBaseAdmin hba = new HBaseAdmin(config);
37+
HTableDescriptor tableDescriptor = null;
38+
HColumnDescriptor columnDescriptor = null;
39+
40+
// Check and optionally create the forward mapping table
41+
if (hba.tableExists(fmap) == false) {
42+
tableDescriptor = new HTableDescriptor(fmap);
43+
columnDescriptor = new HColumnDescriptor("AnonymousValue");
44+
tableDescriptor.addFamily(columnDescriptor);
45+
hba.createTable(tableDescriptor);
46+
}
47+
48+
// Check and optionally create the reverse map table
49+
if (hba.tableExists(rmap) == false) {
50+
tableDescriptor = new HTableDescriptor(rmap);
51+
columnDescriptor = new HColumnDescriptor("ActualValue");
52+
tableDescriptor.addFamily(columnDescriptor);
53+
hba.createTable(tableDescriptor);
54+
}
55+
56+
try {
57+
// Read inputs
58+
String id = (String)input.get(0);
59+
int unique = (int)input.get(1);
60+
System.out.println("Value: " + id + " Unique: " + unique);
61+
Put p = null;
62+
63+
// Generate value for input id
64+
String uuid = UUID.randomUUID().toString();
65+
String hash = DigestUtils.shaHex(id);
66+
67+
System.out.println("UUID: " + uuid + " Hash: " + hash);
68+
69+
// Store the value forward and backward
70+
HTable forward = new HTable(config, fmap);
71+
HTable backward = new HTable(config, rmap);
72+
73+
// Forward Map
74+
p = new Put(Bytes.toBytes(id));
75+
p.add(Bytes.toBytes("AnonymousValue"), Bytes.toBytes("hash"), Bytes.toBytes(hash));
76+
p.add(Bytes.toBytes("AnonymousValue"), Bytes.toBytes("uuid"), Bytes.toBytes(uuid));
77+
forward.put(p);
78+
79+
System.out.println("Stored forward map.");
80+
81+
// Reverse Map - Hash
82+
p = new Put(Bytes.toBytes(hash));
83+
p.add(Bytes.toBytes("ActualValue"), Bytes.toBytes(""), Bytes.toBytes(id));
84+
backward.put(p);
85+
86+
System.out.println("Stored reverse map - hash.");
87+
88+
// Reverse Map - UUID
89+
p = new Put(Bytes.toBytes(uuid));
90+
p.add(Bytes.toBytes("ActualValue"), Bytes.toBytes(""), Bytes.toBytes(id));
91+
backward.put(p);
92+
93+
System.out.println("Stored reverse map - uuid.");
94+
95+
// If the request was for a unique value return the uuid
96+
if (unique == 1) {
97+
return uuid;
98+
} else {
99+
// otherwise return the hashed value
100+
return hash;
101+
}
102+
} catch(Exception e) {
103+
System.out.println(input);
104+
throw new IOException("Caught exception processing input row ", e);
105+
}
106+
}
107+
}

‎DEANONYMIZE.java

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package jaglion;
2+
3+
import java.io.IOException;
4+
import java.util.UUID;
5+
6+
import org.apache.commons.codec.digest.DigestUtils;
7+
8+
import org.apache.pig.EvalFunc;
9+
import org.apache.pig.data.Tuple;
10+
11+
import org.apache.hadoop.conf.Configuration;
12+
import org.apache.hadoop.hbase.HBaseConfiguration;
13+
import org.apache.hadoop.hbase.HTableDescriptor;
14+
import org.apache.hadoop.hbase.HColumnDescriptor;
15+
import org.apache.hadoop.hbase.MasterNotRunningException;
16+
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
17+
import org.apache.hadoop.hbase.client.HBaseAdmin;
18+
import org.apache.hadoop.hbase.client.HTable;
19+
import org.apache.hadoop.hbase.client.Get;
20+
import org.apache.hadoop.hbase.client.Result;
21+
import org.apache.hadoop.hbase.util.Bytes;
22+
23+
public class DEANONYMIZE extends EvalFunc<String>
24+
{
25+
public Configuration config;
26+
public String rmap = "ReversMap";
27+
28+
public String exec(Tuple input) throws IOException {
29+
30+
// Check inputs to ensure there's only one value coming in and that it's not null
31+
if (input == null || input.size() != 1 || input.get(0) == null)
32+
return null;
33+
34+
// Connect to the HBase database
35+
config = HBaseConfiguration.create();
36+
HBaseAdmin hba = new HBaseAdmin(config);
37+
HTableDescriptor tableDescriptor = null;
38+
HColumnDescriptor columnDescriptor = null;
39+
40+
// Ensure the reverse table exists
41+
if (hba.tableExists(rmap) == false) {
42+
throw new IOException("Value not found to map.");
43+
}
44+
45+
try {
46+
String code = (String)input.get(0);
47+
String original = null;
48+
49+
// The backward map table
50+
HTable backward = new HTable(config, rmap);
51+
52+
// Retrieve the value from the reverse map
53+
Get g = new Get(Bytes.toBytes(code));
54+
Result r = backward.get(g);
55+
byte[] value = r.getValue(Bytes.toBytes("ActualValue"), Bytes.toBytes(""));
56+
original = Bytes.toString(value);
57+
58+
return original;
59+
} catch(Exception e) {
60+
System.out.println(input);
61+
throw new IOException("Caught exception processing input row ", e);
62+
}
63+
}
64+
}

‎anonymize.pig

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
-- anonymize.pig
2+
3+
-- -- Hadoop / HBase / Zookeeper
4+
REGISTER /usr/lib/zookeeper/zookeeper.jar;
5+
REGISTER /usr/lib/hadoop/lib/commons-codec-1.4.jar;
6+
REGISTER /usr/lib/hbase/hbase-client.jar;
7+
REGISTER /usr/lib/hbase/hbase-common.jar;
8+
REGISTER /usr/lib/hbase/hbase-protocol.jar;
9+
REGISTER /usr/lib/hbase/hbase-hadoop-compat.jar;
10+
REGISTER /usr/lib/hbase/lib/htrace-core.jar;
11+
12+
-- -- Our UDF
13+
REGISTER bin/jaglion.jar;
14+
15+
-- Load the data with variable length records
16+
-- pass $input to script as -param input='filename'
17+
-- load each line as a character array
18+
A = LOAD '$input' USING TextLoader AS (line:chararray);
19+
20+
-- Split each line into first, rest
21+
SPLT = FOREACH A GENERATE STRSPLIT($0, ',', 2) AS A1;
22+
23+
-- For each row in the data, deanonymize the first column, use the rest as is
24+
B = FOREACH SPLT GENERATE jaglion.ANONYMIZE(TRIM(A1.$0)), A1.$1;
25+
26+
-- Store the deanonymized results out to a file
27+
-- pass $output to script as -param output='filename'
28+
STORE B INTO '$output' using PigStorage(',');

‎deanonymize.pig

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
-- deanonymize.pig
2+
3+
-- -- Hadoop / HBase / Zookeeper
4+
REGISTER /usr/lib/zookeeper/zookeeper.jar;
5+
REGISTER /usr/lib/hadoop/lib/commons-codec-1.4.jar;
6+
REGISTER /usr/lib/hbase/hbase-client.jar;
7+
REGISTER /usr/lib/hbase/hbase-common.jar;
8+
REGISTER /usr/lib/hbase/hbase-protocol.jar;
9+
REGISTER /usr/lib/hbase/hbase-hadoop-compat.jar;
10+
REGISTER /usr/lib/hbase/lib/htrace-core.jar;
11+
12+
-- -- Our UDF
13+
REGISTER bin/jaglion.jar;
14+
15+
-- Load the anonymized data, with variable length records
16+
-- pass $input to script as -param input='filename'
17+
-- load each line as a character array
18+
A = LOAD '$input' USING TextLoader AS (line:chararray);
19+
20+
-- Split each line into first, rest
21+
SPLT = FOREACH A GENERATE STRSPLIT($0, ',', 2) AS A1;
22+
23+
-- For each row in the data, deanonymize the first column, use the rest as is
24+
B = FOREACH SPLT GENERATE jaglion.DEANONYMIZE(TRIM(A1.$0)), A1.$1;
25+
26+
-- Store the deanonymized results out to a file
27+
-- pass $output to script as -param output='filename'
28+
STORE B INTO '$output' using PigStorage(',');

‎test.pig

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- test.pig
2+
REGISTER /usr/lib/zookeeper/zookeeper.jar;
3+
REGISTER /usr/lib/hbase/hbase-client.jar;
4+
REGISTER /usr/lib/hbase/hbase-common.jar;
5+
REGISTER /usr/lib/hbase/hbase-protocol.jar;
6+
REGISTER /usr/lib/hbase/hbase-hadoop-compat.jar;
7+
REGISTER /usr/lib/hbase/lib/htrace-core.jar;
8+
9+
REGISTER bin/jaglion.jar;
10+
A = LOAD 'testdata';
11+
DUMP A;
12+
B = FOREACH A GENERATE jaglion.ANONYMIZE($0, 0);
13+
DUMP B;
14+
-- C = FOREACH A GENERATE jaglion.ANONYMIZE($0, 1);
15+
-- DUMP C;
16+
-- D = FOREACH B GENERATE jaglion.DEANONYMIZE($0);
17+
-- DUMP D;
18+
-- E = FOREACH C GENERATE jaglion.DEANONYMIZE($0);
19+
-- DUMP E;

‎testdata

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Test-String
2+
Test2
3+
IRJ
4+
aaabbb

0 commit comments

Comments
 (0)