Skip to content
This repository was archived by the owner on Oct 30, 2020. It is now read-only.

Commit 9453cdd

Browse files
author
Dmytro Ivchenko
committed
Ported to gradle build system, made tests to work from automation/ide
1 parent 207fdb5 commit 9453cdd

File tree

34 files changed

+747
-619
lines changed

34 files changed

+747
-619
lines changed

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
build
22
build-test
33
dist
4+
out
45
ivy
56
logs
67
output
@@ -28,3 +29,7 @@ target
2829
*.ipr
2930
*.iws
3031
.idea
32+
.gradle
33+
34+
35+

‎build.gradle

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
project.ext.isDefaultEnvironment = !project.hasProperty('overrideBuildEnvironment')
2+
3+
File getEnvironmentScript()
4+
{
5+
final File env = file(isDefaultEnvironment ? 'defaultEnvironment.gradle' : project.overrideBuildEnvironment)
6+
assert env.isFile() : "The environment script [$env] does not exists or is not a file."
7+
return env
8+
}
9+
10+
apply from: environmentScript
11+
12+
project.ext.externalDependency = [
13+
'luceneCore': 'org.apache.lucene:lucene-core:3.5.0',
14+
'luceneAnalyzers': 'org.apache.lucene:lucene-analyzers:3.5.0',
15+
'guava': 'com.google.guava:guava:14.0.1',
16+
'protobufJava': 'com.google.protobuf:protobuf-java:2.4.0a',
17+
'jolokiaCore': 'org.jolokia:jolokia-core:0.90',
18+
'fastutil': 'fastutil:fastutil:5.1.5',
19+
'spring': 'org.springframework:spring:2.5.5',
20+
'log4j': 'log4j:log4j:1.2.16',
21+
'derby': 'org.apache.derby:derby:10.10.1.1',
22+
'zookeeper': 'org.apache.zookeeper:zookeeper:3.3.3',
23+
'httpclient': 'org.apache.httpcomponents:httpclient:4.1.3',
24+
'hadoopCore': 'org.apache.hadoop:hadoop-core:0.20.2',
25+
'commonsCodec': 'commons-codec:commons-codec:1.4',
26+
'commonsLang': 'commons-lang:commons-lang:2.5',
27+
'commonsIo': 'commons-io:commons-io:2.1',
28+
'commonsConfiguration':'commons-configuration:commons-configuration:1.6',
29+
'json':'org.json:json:20080701',
30+
'metricsCore':'com.yammer.metrics:metrics-core:2.0.0-RC0',
31+
'servletApi':'javax.servlet:servlet-api:2.5',
32+
'jetty':'org.mortbay.jetty:jetty:6.1.26',
33+
'javassist':'org.javassist:javassist:3.16.1-GA',
34+
'jdom':'org.jdom:jdom:1.1',
35+
'zkclient': 'com.github.sgroschupf:zkclient:0.1',
36+
37+
'avro': 'org.apache.avro:avro:1.4.0',
38+
'kafka': 'kafka:kafka:0.7.6',
39+
'zoieCore': 'com.linkedin.zoie:zoie-core:3.3.0',
40+
'zoieJms': 'com.linkedin.zoie:zoie-jms:3.3.0',
41+
'boboBrowse': 'com.linkedin.bobo:bobo-browse:3.1.10',
42+
'norbert':'com.linkedin:norbert_2.8.1:0.6.20-SNAPSHOT',
43+
44+
'antlr':'org.antlr:antlr:3.4',
45+
46+
'easymock':'org.easymock:easymock:3.1',
47+
'easymockclassextension':'org.easymock:easymockclassextension:3.1',
48+
'junit':'junit:junit:4.7'
49+
];
50+
51+
subprojects {
52+
}

‎clients/java/build.gradle

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apply plugin: 'java'
2+
3+
dependencies {
4+
compile externalDependency.commonsIo
5+
compile externalDependency.json
6+
compile externalDependency.guava
7+
compile externalDependency.log4j
8+
9+
testCompile externalDependency.junit
10+
}
11+

‎defaultEnvironment.gradle

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
subprojects {
2+
repositories {
3+
mavenCentral()
4+
}
5+
}

‎gradle.properties

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
version=1.5.9

‎sandbox-repo/org/apache/derby/derby/10.10.1.1/derby-10.10.1.1.jar

2.7 MB
Binary file not shown.

‎sensei-core/build.gradle

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
apply plugin: 'java'
2+
3+
configurations {
4+
sourceGen {
5+
}
6+
}
7+
8+
dependencies {
9+
compile externalDependency.luceneCore
10+
compile externalDependency.luceneAnalyzers
11+
compile externalDependency.guava
12+
compile externalDependency.protobufJava
13+
compile externalDependency.jolokiaCore
14+
compile externalDependency.fastutil
15+
compile externalDependency.spring
16+
compile externalDependency.log4j
17+
compile externalDependency.zookeeper
18+
compile externalDependency.httpclient
19+
compile externalDependency.hadoopCore
20+
compile externalDependency.commonsCodec
21+
compile externalDependency.commonsLang
22+
compile externalDependency.commonsIo
23+
compile externalDependency.commonsConfiguration
24+
compile externalDependency.json
25+
compile externalDependency.metricsCore
26+
compile externalDependency.servletApi
27+
compile externalDependency.jetty
28+
compile externalDependency.javassist
29+
compile externalDependency.antlr
30+
31+
compile externalDependency.boboBrowse
32+
compile externalDependency.norbert
33+
compile externalDependency.zoieCore
34+
compile externalDependency.zoieJms
35+
36+
sourceGen externalDependency.antlr
37+
38+
testCompile externalDependency.easymock
39+
testCompile externalDependency.easymockclassextension
40+
testCompile externalDependency.junit
41+
}
42+
43+
task genParser(type: JavaExec) {
44+
ext.src = file('src/main/antlr3/com/senseidb/bql/parsers/BQL.g')
45+
ext.dst = file('src/main/java/com/senseidb/bql/parsers')
46+
47+
inputs.file src
48+
outputs.dir dst
49+
50+
main = 'org.antlr.Tool'
51+
args '-o'
52+
args dst.absolutePath
53+
args src.absolutePath
54+
classpath = configurations.sourceGen
55+
}
56+
57+
compileJava.dependsOn(genParser)

‎sensei-core/src/main/java/com/senseidb/bql/parsers/.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
BQL.tokens
2+
BQLLexer.java
3+
BQLParser.java

‎sensei-core/src/main/java/com/senseidb/gateway/file/LinedFileDataProviderBuilder.java

+23-3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
package com.senseidb.gateway.file;
2020

2121
import java.io.File;
22+
import java.io.IOException;
23+
import java.net.URL;
2224
import java.util.Comparator;
2325
import java.util.Set;
2426

@@ -42,12 +44,30 @@ public StreamDataProvider<JSONObject> buildDataProvider(DataSourceFilter<String>
4244
ShardingStrategy shardingStrategy,
4345
Set<Integer> partitions) throws Exception
4446
{
45-
47+
// either file.path or file.classpath must be set
4648
String path = config.get("file.path");
47-
long offset = oldSinceKey == null ? 0L : Long.parseLong(oldSinceKey);
49+
File file;
50+
if (path != null)
51+
{
52+
file = new File(path);
53+
}
54+
else
55+
{
56+
String classpath = config.get("file.classpath");
57+
if (classpath == null)
58+
throw new IOException("Either file.path or file.classpath must be set");
4859

60+
URL url = LinedFileDataProviderBuilder.class.getClassLoader().getResource(classpath);
61+
if (url == null)
62+
throw new IOException("Resource not found " + path);
63+
file = new File(url.toURI());
64+
}
65+
if (! file.exists())
66+
throw new IOException("File " + path + " not found");
67+
68+
long offset = oldSinceKey == null ? 0L : Long.parseLong(oldSinceKey);
4969

50-
LinedJsonFileDataProvider provider = new LinedJsonFileDataProvider(_versionComparator, new File(path), offset);
70+
LinedJsonFileDataProvider provider = new LinedJsonFileDataProvider(_versionComparator, file, offset);
5171
if (dataFilter!=null){
5272
provider.setFilter(dataFilter);
5373
}

‎sensei-core/src/test/java/com/senseidb/gateway/file/LinedFileDataProviderMockBuilder.java

+23-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
package com.senseidb.gateway.file;
2020

2121
import java.io.File;
22+
import java.io.IOException;
23+
import java.net.URL;
2224
import java.util.Comparator;
2325
import java.util.Set;
2426

@@ -43,11 +45,30 @@ public StreamDataProvider<JSONObject> buildDataProvider(DataSourceFilter<String>
4345
Set<Integer> partitions) throws Exception
4446
{
4547

48+
// either file.path or file.classpath must be set
4649
String path = config.get("file.path");
47-
long offset = oldSinceKey == null ? 0L : Long.parseLong(oldSinceKey);
50+
File file;
51+
if (path != null)
52+
{
53+
file = new File(path);
54+
}
55+
else
56+
{
57+
String classpath = config.get("file.classpath");
58+
if (classpath == null)
59+
throw new IOException("Either file.path or file.classpath must be set");
4860

61+
URL url = LinedFileDataProviderBuilder.class.getClassLoader().getResource(classpath);
62+
if (url == null)
63+
throw new IOException("Resource not found " + path);
64+
file = new File(url.toURI());
65+
}
66+
if (! file.exists())
67+
throw new IOException("File " + path + " not found");
68+
69+
long offset = oldSinceKey == null ? 0L : Long.parseLong(oldSinceKey);
4970

50-
LinedJsonFileDataProvider provider = new FileDataProviderWithMocks(_versionComparator, new File(path), offset);
71+
LinedJsonFileDataProvider provider = new FileDataProviderWithMocks(_versionComparator, file, offset);
5172
if (dataFilter!=null){
5273
provider.setFilter(dataFilter);
5374
}

‎sensei-core/src/test/java/com/senseidb/test/SenseiStarter.java

+61-46
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,18 @@
1919
package com.senseidb.test;
2020

2121
import java.io.File;
22+
import java.io.IOException;
23+
import java.net.InetAddress;
24+
import java.net.InetSocketAddress;
2225
import java.net.URL;
2326

2427
import javax.management.InstanceAlreadyExistsException;
2528

2629
import com.senseidb.indexing.activity.facet.ActivityRangeFacetHandler;
2730
import org.apache.log4j.Logger;
31+
import org.apache.zookeeper.server.NIOServerCnxn;
32+
import org.apache.zookeeper.server.ZooKeeperServer;
33+
import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
2834
import org.mortbay.jetty.Server;
2935
import org.springframework.context.ApplicationContext;
3036
import org.springframework.context.support.ClassPathXmlApplicationContext;
@@ -68,68 +74,76 @@ public class SenseiStarter {
6874
public static SenseiRequestScatterRewriter requestRewriter;
6975
public static NetworkServer networkServer1;
7076
public static NetworkServer networkServer2;
71-
public static final String SENSEI_TEST_CONF_FILE="sensei-test.spring";
7277
public static SenseiZoieFactory<?> _zoieFactory;
7378
public static boolean started = false;
79+
public static URL federatedBrokerUrl;
7480

75-
public static URL federatedBrokerUrl;
81+
private static TestZkServer zkServer = new TestZkServer();
7682

7783

7884
/**
7985
* Will start the new Sensei instance once per process
8086
*/
8187
public static synchronized void start(String confDir1, String confDir2) {
88+
try {
89+
zkServer.start();
90+
} catch (Exception ex) {
91+
logger.error("Failed to start zookeeper", ex);
92+
return;
93+
}
94+
8295
ActivityRangeFacetHandler.isSynchronized = true;
8396
if (started) {
8497
logger.warn("The server had been already started");
8598
return;
8699
}
87100
try {
88101
JmxSenseiMBeanServer.registerCustomMBeanServer();
89-
ConfDir1 = new File(SenseiStarter.class.getClassLoader().getResource(confDir1).toURI());
90-
91-
ConfDir2 = new File(SenseiStarter.class.getClassLoader().getResource(confDir2).toURI());
92-
org.apache.log4j.PropertyConfigurator.configure("resources/log4j.properties");
93-
loadFromSpringContext();
94-
boolean removeSuccessful = rmrf(IndexDir);
95-
if (!removeSuccessful) {
96-
throw new IllegalStateException("The index dir " + IndexDir + " coulnd't be purged");
97-
}
98-
SenseiServerBuilder senseiServerBuilder1 = null;
99-
senseiServerBuilder1 = new SenseiServerBuilder(ConfDir1, null);
100-
node1 = senseiServerBuilder1.buildServer();
101-
httpServer1 = senseiServerBuilder1.buildHttpRestServer();
102-
logger.info("Node 1 created.");
103-
SenseiServerBuilder senseiServerBuilder2 = null;
104-
senseiServerBuilder2 = new SenseiServerBuilder(ConfDir2, null);
105-
node2 = senseiServerBuilder2.buildServer();
106-
httpServer2 = senseiServerBuilder2.buildHttpRestServer();
107-
logger.info("Node 2 created.");
108-
broker = null;
109-
try
110-
{
111-
broker = new SenseiBroker(networkClient, clusterClient, true);
112-
} catch (NorbertException ne) {
113-
logger.info("shutting down cluster...", ne);
114-
clusterClient.shutdown();
115-
throw ne;
116-
}
117-
httpRestSenseiService = new HttpRestSenseiServiceImpl("http", "localhost", 8079, "/sensei");
118-
logger.info("Cluster client started");
119-
Runtime.getRuntime().addShutdownHook(new Thread(){
120-
@Override
121-
public void run(){
122-
shutdownSensei();
123-
}});
124-
node1.start(true);
125-
httpServer1.start();
126-
logger.info("Node 1 started");
127-
node2.start(true);
128-
httpServer2.start();
129-
logger.info("Node 2 started");
130-
SenseiUrl = new URL("http://localhost:8079/sensei");
131-
federatedBrokerUrl = new URL("http://localhost:8079/sensei/federatedBroker/");
132-
waitTillServerStarts();
102+
ConfDir1 = new File(SenseiStarter.class.getClassLoader().getResource(confDir1).toURI());
103+
104+
ConfDir2 = new File(SenseiStarter.class.getClassLoader().getResource(confDir2).toURI());
105+
org.apache.log4j.PropertyConfigurator.configure("resources/log4j.properties");
106+
loadFromSpringContext();
107+
boolean removeSuccessful = rmrf(IndexDir);
108+
if (!removeSuccessful) {
109+
throw new IllegalStateException("The index dir " + IndexDir + " coulnd't be purged");
110+
}
111+
SenseiServerBuilder senseiServerBuilder1 = null;
112+
senseiServerBuilder1 = new SenseiServerBuilder(ConfDir1, null);
113+
node1 = senseiServerBuilder1.buildServer();
114+
httpServer1 = senseiServerBuilder1.buildHttpRestServer();
115+
logger.info("Node 1 created.");
116+
SenseiServerBuilder senseiServerBuilder2 = null;
117+
senseiServerBuilder2 = new SenseiServerBuilder(ConfDir2, null);
118+
node2 = senseiServerBuilder2.buildServer();
119+
httpServer2 = senseiServerBuilder2.buildHttpRestServer();
120+
logger.info("Node 2 created.");
121+
broker = null;
122+
try
123+
{
124+
broker = new SenseiBroker(networkClient, clusterClient, true);
125+
} catch (NorbertException ne) {
126+
logger.info("shutting down cluster...", ne);
127+
clusterClient.shutdown();
128+
throw ne;
129+
}
130+
httpRestSenseiService = new HttpRestSenseiServiceImpl("http", "localhost", 8079, "/sensei");
131+
logger.info("Cluster client started");
132+
Runtime.getRuntime().addShutdownHook(new Thread(){
133+
@Override
134+
public void run(){
135+
shutdownSensei();
136+
}});
137+
node1.start(true);
138+
httpServer1.start();
139+
logger.info("Node 1 started");
140+
node2.start(true);
141+
httpServer2.start();
142+
logger.info("Node 2 started");
143+
SenseiUrl = new URL("http://localhost:8079/sensei");
144+
federatedBrokerUrl = new URL("http://localhost:8079/sensei/federatedBroker/");
145+
146+
waitTillServerStarts();
133147
} catch (Throwable ex) {
134148
logger.error("Could not start the sensei", ex);
135149
throw new RuntimeException(ex);
@@ -198,6 +212,7 @@ private static void shutdownSensei() {
198212
try{networkClient.shutdown();}catch(Throwable t){}
199213
try{clusterClient.shutdown();}catch(Throwable t){}
200214
rmrf(IndexDir);
215+
zkServer.stop();
201216
}
202217

203218
}

0 commit comments

Comments
 (0)