Skip to content

Commit 0a91431

Browse files
qq254963746qq254963746
committed
Merge pull request #12 from qq254963746/develop
抽象 FailStore 接口，后面添加其他实现, 后面计划，优化任务队列模型, 提交处理性能
2 parents 599998d + 404b3df commit 0a91431

File tree

20 files changed

+305
-98
lines changed

20 files changed

+305
-98
lines changed

‎job-client/src/main/java/com/lts/job/client/RetryJobClient.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ public Response submitJob(List<Job> jobs) throws JobSubmitException {
6464
}
6565
response.setSuccess(true);
6666
response.setCode(ResponseCode.FAILED_AND_SAVE_FILE);
67+
response.setMsg(response.getMsg() + ", but save local fail store and send later !");
6768
} catch (Exception e) {
6869
response.setSuccess(false);
6970
response.setMsg(e.getMessage());

‎job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ protected void innerStart() {
3131

3232
NettyRequestProcessor defaultProcessor = getDefaultProcessor();
3333
if (defaultProcessor != null) {
34-
34+
int processorSize = config.getParameter(Constants.PROCESSOR_THREAD, Constants.DEFAULT_PROCESSOR_THREAD);
3535
remotingClient.registerDefaultProcessor(defaultProcessor,
36-
Executors.newFixedThreadPool(32 + Constants.AVAILABLE_PROCESSOR * 5,
36+
Executors.newFixedThreadPool(processorSize,
3737
new NamedThreadFactory(AbstractClientNode.class.getSimpleName())));
3838
}
3939
}
@@ -59,9 +59,9 @@ public void setNodeGroup(String nodeGroup) {
5959
config.setNodeGroup(nodeGroup);
6060
}
6161

62-
public void setJobInfoSavePath(String jobInfoSavePath) {
63-
if (StringUtils.isNotEmpty(jobInfoSavePath)) {
64-
config.setJobInfoSavePath(jobInfoSavePath);
62+
public void setFailStorePath(String failStorePath) {
63+
if (StringUtils.isNotEmpty(failStorePath)) {
64+
config.setFailStorePath(failStorePath);
6565
}
6666
}
6767

‎job-core/src/main/java/com/lts/job/core/cluster/AbstractServerNode.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@ protected void innerStart() {
3030

3131
NettyRequestProcessor defaultProcessor = getDefaultProcessor();
3232
if (defaultProcessor != null) {
33+
int processorSize = config.getParameter(Constants.PROCESSOR_THREAD, Constants.DEFAULT_PROCESSOR_THREAD);
3334
remotingServer.registerDefaultProcessor(defaultProcessor,
34-
Executors.newFixedThreadPool(32 + Constants.AVAILABLE_PROCESSOR * 5, new NamedThreadFactory(AbstractServerNode.class.getSimpleName())));
35+
Executors.newFixedThreadPool(processorSize, new NamedThreadFactory(AbstractServerNode.class.getSimpleName())));
3536
}
3637
}
3738

‎job-core/src/main/java/com/lts/job/core/cluster/Config.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public class Config {
3131
// 监听端口
3232
private int listenPort;
3333
// 任务信息存储路径(譬如TaskTracker反馈任务信息给JobTracker, JobTracker down掉了, 那么存储下来等待JobTracker可用时再发送)
34-
private String jobInfoSavePath;
34+
private String failStorePath;
3535
// 集群名字
3636
private String clusterName;
3737
// java编译器
@@ -105,16 +105,12 @@ public void setListenPort(int listenPort) {
105105
this.listenPort = listenPort;
106106
}
107107

108-
public String getJobInfoSavePath() {
109-
return jobInfoSavePath;
108+
public void setFailStorePath(String failStorePath) {
109+
this.failStorePath = failStorePath;
110110
}
111111

112-
public void setJobInfoSavePath(String jobInfoSavePath) {
113-
this.jobInfoSavePath = jobInfoSavePath + "/.lts";
114-
}
115-
116-
public String getFilePath() {
117-
return jobInfoSavePath + "/" + nodeType + "/" + nodeGroup + "/";
112+
public String getFailStorePath() {
113+
return failStorePath + "/.lts" + "/" + nodeType + "/" + nodeGroup + "/";
118114
}
119115

120116
public boolean isAvailable() {
@@ -140,19 +136,22 @@ public String getParameter(String key, String defaultValue) {
140136
}
141137
return value;
142138
}
139+
143140
private Map<String, Number> getNumbers() {
144141
if (numbers == null) { // 允许并发重复创建
145142
numbers = new ConcurrentHashMap<String, Number>();
146143
}
147144
return numbers;
148145
}
146+
149147
public boolean getParameter(String key, boolean defaultValue) {
150148
String value = getParameter(key);
151149
if (value == null || value.length() == 0) {
152150
return defaultValue;
153151
}
154152
return Boolean.parseBoolean(value);
155153
}
154+
156155
public int getParameter(String key, int defaultValue) {
157156
Number n = getNumbers().get(key);
158157
if (n != null) {
@@ -166,13 +165,15 @@ public int getParameter(String key, int defaultValue) {
166165
getNumbers().put(key, i);
167166
return i;
168167
}
168+
169169
public String[] getParameter(String key, String[] defaultValue) {
170170
String value = getParameter(key);
171171
if (value == null || value.length() == 0) {
172172
return defaultValue;
173173
}
174174
return Constants.COMMA_SPLIT_PATTERN.split(value);
175175
}
176+
176177
public double getParameter(String key, double defaultValue) {
177178
Number n = getNumbers().get(key);
178179
if (n != null) {
@@ -186,6 +187,7 @@ public double getParameter(String key, double defaultValue) {
186187
getNumbers().put(key, d);
187188
return d;
188189
}
190+
189191
public float getParameter(String key, float defaultValue) {
190192
Number n = getNumbers().get(key);
191193
if (n != null) {

‎job-core/src/main/java/com/lts/job/core/constant/Constants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,7 @@ public interface Constants {
7070
// 客户端提交并发请求size
7171
public static final String JOB_SUBMIT_CONCURRENCY_SIZE = "job.submit.concurrency.size";
7272
public static final int DEFAULT_JOB_SUBMIT_CONCURRENCY_SIZE = Constants.AVAILABLE_PROCESSOR * 4;
73+
74+
public static final String PROCESSOR_THREAD = "job.processor.thread";
75+
public static final int DEFAULT_PROCESSOR_THREAD = 32 + AVAILABLE_PROCESSOR * 5;
7376
}

‎job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public static Config getDefaultConfig() {
1717
config.setRegistryAddress("zookeeper://127.0.0.1:2181");
1818
config.setInvokeTimeoutMillis(1000 * 6);
1919
config.setListenPort(0);
20-
config.setJobInfoSavePath(Constants.USER_HOME);
20+
config.setFailStorePath(Constants.USER_HOME);
2121
config.setClusterName(Constants.DEFAULT_CLUSTER_NAME);
2222
return config;
2323
}

‎job-core/src/main/java/com/lts/job/core/failstore/FailStore.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package com.lts.job.core.failstore;
2+
3+
import com.lts.job.core.domain.KVPair;
4+
5+
import java.lang.reflect.Type;
6+
import java.util.List;
7+
8+
/**
9+
* Created by hugui on 5/21/15.
10+
*/
11+
public interface FailStore {
12+
13+
public void open() throws FailStoreException;
14+
15+
public void put(String key, Object value) throws FailStoreException;
16+
17+
public void delete(String key) throws FailStoreException;
18+
19+
public void delete(List<String> keys) throws FailStoreException;
20+
21+
public <T> List<KVPair<String, T>> fetchTop(int size, Type type) throws FailStoreException;
22+
23+
public void close() throws FailStoreException;
24+
25+
public void destroy() throws FailStoreException;
26+
}

‎job-core/src/main/java/com/lts/job/core/failstore/FailStoreException.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package com.lts.job.core.failstore;
2+
3+
/**
4+
* Created by hugui on 5/21/15.
5+
*/
6+
public class FailStoreException extends Exception {
7+
8+
public FailStoreException(String message) {
9+
super(message);
10+
}
11+
12+
public FailStoreException(String message, Throwable cause) {
13+
super(message, cause);
14+
}
15+
16+
public FailStoreException(Throwable cause) {
17+
super(cause);
18+
}
19+
20+
protected FailStoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
21+
super(message, cause, enableSuppression, writableStackTrace);
22+
}
23+
}

‎job-core/src/main/java/com/lts/job/core/failstore/FailStoreFactory.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package com.lts.job.core.failstore;
2+
3+
import com.lts.job.core.cluster.Config;
4+
import com.lts.job.core.extension.Adaptive;
5+
import com.lts.job.core.extension.SPI;
6+
7+
/**
8+
* Created by hugui on 5/21/15.
9+
*/
10+
@SPI("leveldb")
11+
public interface FailStoreFactory {
12+
13+
@Adaptive("job.fail.store")
14+
public FailStore getFailStore(Config config);
15+
16+
}

‎job-core/src/main/java/com/lts/job/core/failstore/leveldb/LeveldbFailStore.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package com.lts.job.core.failstore.leveldb;
2+
3+
import com.lts.job.core.cluster.Config;
4+
import com.lts.job.core.domain.KVPair;
5+
import com.lts.job.core.failstore.FailStore;
6+
import com.lts.job.core.failstore.FailStoreException;
7+
import com.lts.job.core.file.FileAccessor;
8+
import com.lts.job.core.file.FileException;
9+
import com.lts.job.core.file.FileUtils;
10+
import com.lts.job.core.util.JSONUtils;
11+
import org.fusesource.leveldbjni.JniDBFactory;
12+
import org.iq80.leveldb.DB;
13+
import org.iq80.leveldb.DBIterator;
14+
import org.iq80.leveldb.Options;
15+
16+
import java.io.File;
17+
import java.io.IOException;
18+
import java.lang.reflect.Type;
19+
import java.util.ArrayList;
20+
import java.util.List;
21+
22+
/**
23+
* Created by hugui on 5/21/15.
24+
*/
25+
public class LeveldbFailStore implements FailStore {
26+
27+
// 文件锁 (同一时间只能有一个线程在 检查提交失败的任务)
28+
private FileAccessor dbLock;
29+
/**
30+
* 数据库目录
31+
*/
32+
private File dbPath;
33+
34+
private DB db;
35+
36+
private Options options;
37+
38+
public LeveldbFailStore(Config config) {
39+
dbPath = FileUtils.createDirIfNotExist(config.getFailStorePath());
40+
options = new Options();
41+
try {
42+
dbLock = new FileAccessor(config.getFailStorePath() + "___db.lock");
43+
dbLock.createIfNotExist();
44+
} catch (FileException e) {
45+
throw new RuntimeException(e);
46+
}
47+
}
48+
49+
@Override
50+
public void open() throws FailStoreException {
51+
dbLock.tryLock();
52+
try {
53+
db = JniDBFactory.factory.open(dbPath, options);
54+
} catch (IOException e) {
55+
throw new FailStoreException(e);
56+
}
57+
}
58+
59+
@Override
60+
public void put(String key, Object value) throws FailStoreException {
61+
String valueString = JSONUtils.toJSONString(value);
62+
db.put(key.getBytes(), valueString.getBytes());
63+
}
64+
65+
@Override
66+
public void delete(String key) throws FailStoreException {
67+
if (key == null) {
68+
return;
69+
}
70+
db.delete(key.getBytes());
71+
}
72+
73+
@Override
74+
public void delete(List<String> keys) throws FailStoreException {
75+
if (keys == null || keys.size() == 0) {
76+
return;
77+
}
78+
for (String key : keys) {
79+
delete(key);
80+
}
81+
}
82+
83+
@Override
84+
public <T> List<KVPair<String, T>> fetchTop(int size, Type type) {
85+
List<KVPair<String, T>> list = new ArrayList<KVPair<String, T>>(size);
86+
DBIterator iterator = db.iterator();
87+
for (iterator.seekToLast(); iterator.hasPrev(); iterator.prev()) {
88+
String key = new String(iterator.peekPrev().getKey());
89+
T value = JSONUtils.parse(new String(iterator.peekPrev().getValue()), type);
90+
KVPair<String, T> pair = new KVPair<String, T>(key, value);
91+
list.add(pair);
92+
if (list.size() >= size) {
93+
break;
94+
}
95+
}
96+
return list;
97+
}
98+
99+
@Override
100+
public void close() throws FailStoreException {
101+
try {
102+
db.close();
103+
dbLock.unlock();
104+
} catch (IOException e) {
105+
throw new FailStoreException(e);
106+
}
107+
}
108+
109+
public void destroy() throws FailStoreException {
110+
try {
111+
JniDBFactory.factory.destroy(dbPath, options);
112+
dbLock.delete();
113+
} catch (IOException e) {
114+
throw new FailStoreException(e);
115+
}
116+
}
117+
}

0 commit comments

Comments
 (0)