From c5ab96921d4ed1d5b0e5ed9e9cb9d4693c627565 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 17 Oct 2022 10:57:23 +0800 Subject: [PATCH 001/219] Support the method to get engine conn resource information. --- pom.xml | 2 +- .../url/LinkisURLStreamHandlerFactory.java | 1 - .../linkis/core/FlinkLogIterator.scala | 6 ++ .../linkis/entity/LogRequestPayload.scala | 5 ++ .../launcher/linkis/job/FlinkJobClient.scala | 67 +++++++++++++++++-- .../launcher/linkis/job/FlinkJobInfo.scala | 30 +++++++-- .../launcher/linkis/job/LinkisJobInfo.scala | 6 ++ .../manager/SimpleFlinkJobLaunchManager.scala | 9 +-- .../job/operator/FlinkClientLogOperator.scala | 34 ++++++++++ .../job/operator/FlinkYarnLogOperator.scala | 7 +- .../service/DefaultStreamTaskService.scala | 15 ++++- 11 files changed, 160 insertions(+), 22 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala diff --git a/pom.xml b/pom.xml index eabaaba66..bd535bf6f 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ - 1.1.1 + 1.1.3 1.1.0 0.2.0 2.11.12 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java index 78701f646..1352c8b28 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java @@ -15,7 +15,6 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.url; -import org.apache.commons.lang.StringUtils; import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala index 2fdb12b7b..d6fb5c661 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala @@ -18,6 +18,7 @@ trait FlinkLogIterator extends Iterator[String] with Closeable { val engineConnLogOperator: EngineConnLogOperator def init(): Unit def getLogPath: String + def getLogDirSuffix: String def getLogs: util.ArrayList[String] def getEndLine: Long } @@ -28,6 +29,7 @@ class SimpleFlinkJobLogIterator(override val requestPayload: LogRequestPayload, private var logs: util.ArrayList[String] = _ private var index = 0 private var logPath: String = _ + private var logDirSuffix: String = _ private var isClosed = true private var endLine = 0 @@ -69,4 +71,8 @@ class SimpleFlinkJobLogIterator(override val requestPayload: LogRequestPayload, override def getLogs: util.ArrayList[String] = logs override def getEndLine: Long = endLine + + def setLogDirSuffix(logDirSuffix: String) : Unit = this.logDirSuffix = logDirSuffix + + override def getLogDirSuffix: String = logDirSuffix } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala index 29f90d325..20cb4d081 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala @@ -14,6 +14,7 @@ class LogRequestPayload { private var onlyKeywords: String = _ private var lastRows = 0 private var logType: String = _ + private var logHistory: Boolean = false def getPageSize: Int = pageSize def setPageSize(pageSize: Int): Unit = this.pageSize = pageSize @@ -32,4 +33,8 @@ class LogRequestPayload { def getLogType: String = logType def setLogType(logType: String): Unit = this.logType = logType + + def isLogHistory: Boolean = logHistory + + def setLogHistory(logHistory: Boolean): Unit = this.logHistory = logHistory } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala index 951145b94..b5c64a523 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala @@ -23,13 +23,17 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.core.{FlinkLo import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.exception.{FlinkJobLaunchErrorException, FlinkJobStateFetchException, FlinkSavePointException} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.manager.FlinkJobLaunchManager -import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.operator.{FlinkTriggerSavepointOperator, FlinkYarnLogOperator} +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.operator.{FlinkClientLogOperator, FlinkTriggerSavepointOperator, FlinkYarnLogOperator} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.{Checkpoint, Savepoint} +import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.computation.client.once.OnceJob -import org.apache.linkis.computation.client.once.simple.SimpleOnceJob +import org.apache.linkis.computation.client.once.action.ECResourceInfoAction +import org.apache.linkis.computation.client.once.result.ECResourceInfoResult +import org.apache.linkis.computation.client.once.{LinkisManagerClient, LinkisManagerClientImpl, OnceJob} +import org.apache.linkis.computation.client.once.simple.{SimpleOnceJob, SimpleOnceJobBuilder} import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator - +import org.apache.linkis.httpclient.dws.DWSHttpClient +import java.util import java.net.URI class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: JobStateManager) @@ -39,9 +43,13 @@ class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: * Log operator */ private var logOperatorMap = Map( - "client" -> EngineConnLogOperator.OPERATOR_NAME, + "client" -> FlinkClientLogOperator.OPERATOR_NAME, "yarn" -> FlinkYarnLogOperator.OPERATOR_NAME ) + /** + * The linkis client in onceJob + */ + private var linkisClient: DWSHttpClient = _ override def getJobInfo: FlinkJobInfo = { getJobInfo(false) @@ -99,13 +107,38 @@ class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: case Some(operator) => onceJob.getOperator(operator) match { case engineConnLogOperator: EngineConnLogOperator => + val logIterator = new SimpleFlinkJobLogIterator(requestPayload, engineConnLogOperator) + engineConnLogOperator match { + case clientLogOperator: FlinkClientLogOperator => + var logDirSuffix = this.jobInfo.getLogDirSuffix + if (StringUtils.isBlank(logDirSuffix) && requestPayload.isLogHistory){ + // If want to fetch the history log, must get the log directory suffix first + getLinkisClient match { + case client: DWSHttpClient => + Option(Utils.tryCatch{ + client.execute(ECResourceInfoAction.newBuilder().setUser(jobInfo.getUser) + .setTicketid(clientLogOperator.getTicketId).build()).asInstanceOf[ECResourceInfoResult] + }{ + case e: Exception => + warn("Fail to query the engine conn resource info from linkis", e) + null + }) match { + case Some(result) => logDirSuffix = Utils.tryAndWarn{result.getData.getOrDefault("ecResourceInfoRecord", new util.HashMap[String, Any]).asInstanceOf[util.Map[String, Any]] + .getOrDefault("logDirSuffix", "").asInstanceOf[String]} + case _ => + } + } + } + clientLogOperator.setLogDirSuffix(logDirSuffix) + logIterator.setLogDirSuffix(logDirSuffix) + case _ => + } engineConnLogOperator match { case yarnLogOperator: FlinkYarnLogOperator => yarnLogOperator.setApplicationId(jobInfo.getApplicationId) case _ => } engineConnLogOperator.setECMServiceInstance(jobInfo.getECMInstance) engineConnLogOperator.setEngineConnType(FlinkJobLaunchManager.FLINK_ENGINE_CONN_TYPE) - val logIterator = new SimpleFlinkJobLogIterator(requestPayload, engineConnLogOperator) logIterator.init() jobInfo match { case jobInfo: FlinkJobInfo => jobInfo.setLogPath(logIterator.getLogPath) @@ -161,5 +194,27 @@ class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: triggerSavepoint(savepointURI.toString, JobLauncherConfiguration.FLINK_TRIGGER_SAVEPOINT_MODE.getValue) } + /** + * Get linkis client + * @return + */ + def getLinkisClient: DWSHttpClient = { + Utils.tryAndWarn{ + if (null == this.linkisClient){ + this.synchronized{ + if (null == this.linkisClient){ + this.linkisClient = SimpleOnceJobBuilder.getLinkisManagerClient match { + case client: LinkisManagerClient => + val dwsClientField = classOf[LinkisManagerClientImpl].getDeclaredField("dwsHttpClient") + dwsClientField.setAccessible(true) + dwsClientField.get(client).asInstanceOf[DWSHttpClient] + case _ => null + } + } + } + } + this.linkisClient + } + } } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala index 5c2986609..0fd1f1783 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala @@ -34,41 +34,53 @@ class FlinkJobInfo extends YarnJobInfo { private var applicationUrl: String = _ private var status: String = _ private var logPath: String = _ + private var logDirSuffix: String = _ private var resources: java.util.Map[String, Object] = _ private var completedMsg: String = _ private var jobStates: Array[JobStateInfo] = _ + override def getApplicationId: String = applicationId + def setApplicationId(applicationId: String): Unit = this.applicationId = applicationId override def getApplicationUrl: String = applicationUrl + def setApplicationUrl(applicationUrl: String): Unit = this.applicationUrl = applicationUrl override def getId: String = id - def setId(id: String): Unit = this.id = id + def setId(id: String): Unit = this.id = id override def getECMInstance: ServiceInstance = ecmInstance + def setECMInstance(ecmInstance: ServiceInstance): Unit = this.ecmInstance = ecmInstance override def getUser: String = user + def setUser(user: String): Unit = this.user = user override def getStatus: String = status + override def setStatus(status: String): Unit = this.status = status override def getLogPath: String = logPath + def setLogPath(logPath: String): Unit = this.logPath = logPath override def getResources: util.Map[String, Object] = resources + def setResources(resources: java.util.Map[String, Object]): Unit = this.resources = resources def getSavepoint: String = savepoint + def setSavepoint(savepoint: String): Unit = this.savepoint = savepoint def getCheckpoint: String = checkpoint + def setCheckpoint(checkpoint: String): Unit = this.checkpoint = checkpoint override def getCompletedMsg: String = completedMsg + def setCompletedMsg(completedMsg: String): Unit = this.completedMsg = completedMsg override def toString: String = s"FlinkJobInfo(id: $id, status: $status, applicationId: $applicationId, applicationUrl: $applicationUrl, logPath: $logPath)" @@ -85,6 +97,7 @@ class FlinkJobInfo extends YarnJobInfo { def setJobStates(jobStates: Array[JobStateInfo]): Unit = { this.jobStates = jobStates } + /** * Job name * @@ -95,11 +108,16 @@ class FlinkJobInfo extends YarnJobInfo { def setName(name: String): Unit = { this.name = name } -} -object FlinkJobInfo{ - def main(args: Array[String]): Unit = { - val jobInfo = "{\"jobStates:\":{\"location\":\"xx\"}" - DWSHttpClient.jacksonJson.readValue(jobInfo, classOf[FlinkJobInfo]) + /** + * Job log directory suffix + * + * @return + */ + override def getLogDirSuffix: String = this.logDirSuffix + + override def setLogDirSuffix(logDirSuffix: String): Unit = { + this.logDirSuffix = logDirSuffix } } + diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala index da02fda50..4183f5025 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala @@ -11,5 +11,11 @@ trait LinkisJobInfo extends JobInfo { */ def getECMInstance: ServiceInstance + /** + * Job log directory suffix + * @return + */ + def getLogDirSuffix: String + def setLogDirSuffix(logDirSuffix: String): Unit } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala index 6ad2e4f88..5e69766fe 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala @@ -15,16 +15,16 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.manager -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.{JobState, JobStateInfo} +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobState import com.webank.wedatasphere.streamis.jobmanager.launcher.job.{JobClient, LaunchJob} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.conf.JobLauncherConfiguration -import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo, LinkisJobInfo} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.manager.SimpleFlinkJobLaunchManager.INSTANCE_NAME -import org.apache.commons.lang.StringEscapeUtils +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo, LinkisJobInfo} +import org.apache.commons.lang3.StringEscapeUtils import org.apache.linkis.common.utils.{RetryHandler, Utils} import org.apache.linkis.computation.client.once.simple.{SimpleOnceJob, SubmittableSimpleOnceJob} import org.apache.linkis.computation.client.once.{OnceJob, SubmittableOnceJob} -import org.apache.linkis.computation.client.operator.impl.{EngineConnApplicationInfoOperator, EngineConnLogOperator} +import org.apache.linkis.computation.client.operator.impl.EngineConnApplicationInfoOperator import org.apache.linkis.httpclient.dws.DWSHttpClient import org.apache.linkis.ujes.client.exception.UJESJobException @@ -69,6 +69,7 @@ class SimpleFlinkJobLaunchManager extends FlinkJobLaunchManager { jobInfo.setCompletedMsg(message) } jobInfo.setResources(nodeInfo.get("nodeResource").asInstanceOf[util.Map[String, Object]]) + jobInfo.setStatus("failed") // Set job state info into // Option(jobState).foreach(state => { // val stateInfo = new JobStateInfo diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala new file mode 100644 index 000000000..a41018a74 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.operator + +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.computation.client.once.action.EngineConnOperateAction +import org.apache.linkis.computation.client.operator.impl.{EngineConnLogOperator, EngineConnLogs} + +/** + * Append "logDirSuffix" parameter + */ +class FlinkClientLogOperator extends EngineConnLogOperator{ + + private var logDirSuffix: String = _ + + def setLogDirSuffix(logDirSuffix: String): Unit = { + this.logDirSuffix = logDirSuffix + } + + protected override def addParameters(builder: EngineConnOperateAction.Builder): Unit = { + builder.operatorName(EngineConnLogOperator.OPERATOR_NAME) + if (StringUtils.isNotBlank(this.logDirSuffix)) { + builder.addParameter("logDirSuffix", logDirSuffix) + } + super.addParameters(builder) + } + + + override def getTicketId: String = super.getTicketId + + override def getName: String = FlinkClientLogOperator.OPERATOR_NAME +} + +object FlinkClientLogOperator { + val OPERATOR_NAME = "engineConnLog_flink" +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala index a24e12580..975b23405 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala @@ -19,9 +19,9 @@ import org.apache.linkis.computation.client.once.action.EngineConnOperateAction import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator /** - * Extend the engine conn log operator + * Extend the flink client log operator */ -class FlinkYarnLogOperator extends EngineConnLogOperator{ +class FlinkYarnLogOperator extends FlinkClientLogOperator { private var applicationId: String = _ @@ -30,8 +30,9 @@ class FlinkYarnLogOperator extends EngineConnLogOperator{ } protected override def addParameters(builder: EngineConnOperateAction.Builder): Unit = { - builder.addParameter("yarnApplicationId", this.applicationId) super.addParameters(builder) + builder.operatorName(getName) + builder.addParameter("yarnApplicationId", this.applicationId) } override def getName: String = FlinkYarnLogOperator.OPERATOR_NAME diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index 1913fc0c9..5e7d065f7 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -22,7 +22,7 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobState import com.webank.wedatasphere.streamis.jobmanager.launcher.job.{JobInfo, LaunchJob} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.{Checkpoint, Savepoint} -import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo} +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo, LinkisJobInfo} import com.webank.wedatasphere.streamis.jobmanager.manager.SpringContextHolder import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf.FLINK_JOB_STATUS_FAILED @@ -398,11 +398,24 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ val jobClient = jobLaunchManager.connect(streamTask.getLinkisJobId, streamTask.getLinkisJobInfo) jobClient match { case client: FlinkJobClient => + requestPayload.setLogHistory(JobConf.isCompleted(streamTask.getStatus)) val logIterator = client.fetchLogs(requestPayload) returnMap.put("logPath", logIterator.getLogPath) returnMap.put("logs", logIterator.getLogs) returnMap.put("endLine", logIterator.getEndLine) logIterator.close() + jobClient.getJobInfo match { + case linkisInfo: LinkisJobInfo => + if (StringUtils.isBlank(linkisInfo.getLogDirSuffix) && StringUtils.isNotBlank(logIterator.getLogDirSuffix)){ + Utils.tryAndWarn { + // Update the linkis job info and store into database + linkisInfo.setLogDirSuffix(logIterator.getLogDirSuffix) + streamTask.setLinkisJobInfo(DWSHttpClient.jacksonJson.writeValueAsString(linkisInfo)); + streamTaskMapper.updateTask(streamTask) + } + } + case _ => + } } }{ case e: Exception => // Just warn the exception From 9f6b0be451e443de63c22bfe5285f598d8eb7a51 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 17 Oct 2022 10:58:10 +0800 Subject: [PATCH 002/219] Init the structure of log module. --- streamis-jobmanager/pom.xml | 1 + .../flink-streamis-log-collector/pom.xml | 28 +++++++++ streamis-jobmanager/streamis-job-log/pom.xml | 26 +++++++++ .../streamis-job-log-collector/pom.xml | 19 ++++++ .../streamis-job-log-common/pom.xml | 19 ++++++ .../jobmanager/log/entities/LogElement.java | 29 ++++++++++ .../log/entities/StreamisLogEvent.java | 4 ++ .../streamis-job-log-server/pom.xml | 26 +++++++++ .../StreamisJobLogAutoConfiguration.java | 4 ++ .../log/server/storage/JobLogStorage.java | 18 ++++++ .../server/storage/StreamisJobLogStorage.java | 4 ++ .../server/storage/bucket/JobLogBucket.java | 24 ++++++++ .../storage/bucket/JobLogBucketConfig.java | 58 +++++++++++++++++++ .../storage/bucket/JobLogBucketFactory.java | 15 +++++ .../storage/bucket/JobLogBucketState.java | 19 ++++++ .../storage/bucket/JobLogStorageWriter.java | 27 +++++++++ .../storage/bucket/Log4j2JobLogBucket.java | 4 ++ .../storage/bucket/StreamisJobLogBucket.java | 7 +++ 18 files changed, 332 insertions(+) create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java diff --git a/streamis-jobmanager/pom.xml b/streamis-jobmanager/pom.xml index b1a644e09..e88713dc2 100644 --- a/streamis-jobmanager/pom.xml +++ b/streamis-jobmanager/pom.xml @@ -32,6 +32,7 @@ streamis-job-manager streamis-jobmanager-server streamis-projectmanager-server + streamis-job-log diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml new file mode 100644 index 000000000..5207e035a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml @@ -0,0 +1,28 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + flink-streamis-log-collector + + + 8 + 8 + + 1.12.2 + + + + + org.apache.flink + flink-java + ${flink.version} + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml new file mode 100644 index 000000000..3446a972b --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -0,0 +1,26 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log + pom + + streamis-job-log-collector + flink-streamis-log-collector + streamis-job-log-server + streamis-job-log-common + + + + 8 + 8 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml new file mode 100644 index 000000000..aa20062f4 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -0,0 +1,19 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-collector + + + 8 + 8 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml new file mode 100644 index 000000000..51d283f53 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -0,0 +1,19 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-common + + + 8 + 8 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java new file mode 100644 index 000000000..a2b0a7807 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.entities; + +import java.util.concurrent.TimeUnit; + +/** + * Element defined of log + */ +public interface LogElement { + + /** + * Sequence id + * @return seq id + */ + int getSequenceId(); + + /** + * Log time + * @param unit unit + * @return log time + */ + long getLogTime(TimeUnit unit); + + /** + * Get content + * @return content array + */ + String[] getContents(); + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java new file mode 100644 index 000000000..0c73d83d2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.entities; + +public class StreamisLogEvent { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml new file mode 100644 index 000000000..69936a7b4 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -0,0 +1,26 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-service + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-log-common + 0.2.0 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java new file mode 100644 index 000000000..7b839893a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server; + +public class StreamisJobLogAutoConfiguration { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java new file mode 100644 index 000000000..e951a1e7a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage; + +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketConfig; + +/** + * Storage of job log + */ +public interface JobLogStorage { + + /** + * Create buckets + * @param jobName job name + * @param bucketConfig bucket config + * @return config + */ + JobLogBucket getOrCreateBucket(String jobName, JobLogBucketConfig bucketConfig); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java new file mode 100644 index 000000000..b357c30c6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage; + +public class StreamisJobLogStorage { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java new file mode 100644 index 000000000..c20c08944 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * Job log bucket for streamis + */ +public interface JobLogBucket { + + /** + * Bucket state + * @return state + */ + JobLogBucketState getBucketState(); + + /** + * Storage writer + * @return storage writer + */ + JobLogStorageWriter getBucketStorageWriter(); + + /** + * Close the bucket + */ + void close(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java new file mode 100644 index 000000000..85a5c2d2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java @@ -0,0 +1,58 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +import java.util.HashMap; +import java.util.Map; + +/** + * Configuration for job log bucket + */ +public class JobLogBucketConfig { + + /** + * Bucket class + */ + private Class bucketClass; + + /** + * Attribute + */ + protected Map attributes = new HashMap<>(); + + /** + * Max size of bucket part + */ + private int maxBucketPartSize; + + /** + * Layout pattern + */ + private String LayOutPattern; + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + this.attributes = attributes; + } + + public int getMaxBucketPartSize() { + return maxBucketPartSize; + } + + public void setMaxBucketPartSize(int maxBucketPartSize) { + this.maxBucketPartSize = maxBucketPartSize; + } + + public String getLayOutPattern() { + return LayOutPattern; + } + + public void setLayOutPattern(String layOutPattern) { + LayOutPattern = layOutPattern; + } + + public static final class Define{ + + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java new file mode 100644 index 000000000..d4b9b6b2a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * Factory of creating job log bucket + */ +public interface JobLogBucketFactory { + + /** + * Create bucket + * @param jobName job name + * @param config bucket config + * @return + */ + JobLogBucket createBucket(String jobName, JobLogBucketConfig config); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java new file mode 100644 index 000000000..bdd64f265 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * State of log bucket + */ +public interface JobLogBucketState { + + /** + * Bucket path + * @return path + */ + String getBucketPath(); + + double getBucketWriteRate(); + + int getBucketParts(); + + int getBucketWriteTime(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java new file mode 100644 index 000000000..772040374 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +/** + * Storage writer for job log + */ +public interface JobLogStorageWriter { + + /** + * Write log element + * @param logEl elements + * @param + */ + void write(LogElement logEl); + + /** + * Write log line + * @param logLine log line + */ + void write(String logLine); + + /** + * Close log storage + */ + void close(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java new file mode 100644 index 000000000..21c3303f5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +public class Log4j2JobLogBucket { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java new file mode 100644 index 000000000..ba9c002d6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * Use the appender and strategy of log4j (version 1.x) to implement the bucket + */ +public class StreamisJobLogBucket { +} From a04b58d151c0d2d151dbcc91bb28c2e5c098ea9b Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 18 Oct 2022 18:23:21 +0800 Subject: [PATCH 003/219] Add the new feature for adding task and stop task for existed jobs. --- .../manager/dao/StreamJobMapper.java | 1 + .../manager/dao/impl/StreamJobMapper.xml | 5 + .../service/DefaultStreamJobService.scala | 10 +- .../service/DefaultStreamTaskService.scala | 7 +- .../manager/service/StreamJobService.scala | 3 + .../manager/service/StreamTaskService.scala | 7 +- .../jobmanager/restful/api/JobRestfulApi.java | 110 +++++++++++++++++- 7 files changed, 131 insertions(+), 12 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java index c380ca954..3e002c45d 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -30,6 +30,7 @@ List getJobLists(@Param("projectName") String projectName, @Para StreamJob getJobById(@Param("jobId") Long jobId); + List getJobByName(@Param("jobName") String jobName); List getJobVersions(@Param("jobId") Long jobId); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index 71e2b38a6..875da3e4f 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -86,6 +86,11 @@ + + diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index c3f467118..f12878433 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -17,26 +17,26 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import java.util import java.util.Date + import com.github.pagehelper.PageInfo import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService import com.webank.wedatasphere.streamis.jobmanager.manager.alert.AlertLevel -import org.apache.linkis.common.exception.ErrorException -import org.apache.linkis.common.utils.Logging import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamAlertMapper, StreamJobMapper, StreamTaskMapper} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{QueryJobListVo, TaskCoreNumVo, VersionDetailVo} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{MetaJsonInfo, StreamAlertRecord, StreamJob, StreamJobVersion, StreamJobVersionFiles} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity._ import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobCreateErrorException, JobFetchErrorException} import com.webank.wedatasphere.streamis.jobmanager.manager.transform.JobContentParser import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent import com.webank.wedatasphere.streamis.jobmanager.manager.util.{ReaderUtils, ZipHelper} import org.apache.commons.lang.StringUtils +import org.apache.linkis.common.exception.ErrorException +import org.apache.linkis.common.utils.Logging import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional -import javax.annotation.Resource import scala.collection.JavaConverters._ @@ -60,6 +60,8 @@ class DefaultStreamJobService extends StreamJobService with Logging { this.streamJobMapper.getJobById(jobId) } + override def getJobByName(jobName: String): util.List[StreamJob] = streamJobMapper.getJobByName(jobName) + override def getByProList(projectName: String, userName: String, jobName: String, jobStatus: Integer, jobCreator: String): PageInfo[QueryJobListVo] = { val streamJobList = streamJobMapper.getJobLists(projectName, userName, jobName, jobStatus, jobCreator) if (streamJobList != null && !streamJobList.isEmpty) { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index 1913fc0c9..099866004 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -470,7 +470,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ }).asJava } - def getTask(jobId:Long, version: String): FlinkJobInfo ={ + def getTaskJobInfo(jobId:Long, version: String): FlinkJobInfo ={ val str = streamTaskMapper.getTask(jobId, version) if (StringUtils.isBlank(str)) { return new FlinkJobInfo @@ -511,6 +511,9 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ } } + + override def getLatestTaskByJobId(jobId: Long): StreamTask = streamTaskMapper.getLatestByJobId(jobId) + /** * Create new task use the latest job version * @@ -545,6 +548,8 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ } } + override def updateTask(streamTask: StreamTask): Unit = streamTaskMapper.updateTask(streamTask) + /** * Just launch task by task id * diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala index 5f27f0864..5ac798177 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala @@ -15,6 +15,9 @@ trait StreamJobService { def getJobById(jobId: Long): StreamJob + + def getJobByName(jobName: String): util.List[StreamJob] + /** * Page list query * @param projectName project name diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala index 079a8d8f8..65c584804 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala @@ -20,7 +20,6 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogReq import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobProgressVo, JobStatusVo, PauseResultVo, StreamTaskListVo} - import java.util import java.util.concurrent.Future /** @@ -87,6 +86,8 @@ trait StreamTaskService { */ def launch(taskId: Long, execUser: String): Unit + def getLatestTaskByJobId(jobId: Long): StreamTask + /** * Create new task use the latest job version * @param jobId job id @@ -95,6 +96,8 @@ trait StreamTaskService { */ def createTask(jobId: Long, status: Int, creator: String): StreamTask + def updateTask(streamTask: StreamTask): Unit + /** * Update the task status * @param jobId job id @@ -146,7 +149,7 @@ trait StreamTaskService { * @param version version * @return */ - def getTask(jobId: Long, version: String): FlinkJobInfo + def getTaskJobInfo(jobId: Long, version: String): FlinkJobInfo def getStateInfo(taskId: Long): JobState diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 6d6591d74..c6c9b5a97 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -15,6 +15,7 @@ package com.webank.wedatasphere.streamis.jobmanager.restful.api; +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.webank.wedatasphere.streamis.jobmanager.exception.JobException; @@ -22,16 +23,22 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.JobInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload; +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo; +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.MetaJsonInfo; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.*; import com.webank.wedatasphere.streamis.jobmanager.manager.project.service.ProjectPrivilegeService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamTaskService; import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent; +import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.httpclient.dws.DWSHttpClient; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -43,10 +50,7 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; @RequestMapping(path = "/streamis/streamJobManager/job") @RestController @@ -210,7 +214,7 @@ public Message detailsJob(HttpServletRequest req, @RequestParam(value = "jobId", realTimeTrafficDTOS.add(realTimeTrafficDTO); - jobDetailsVO.setLinkisJobInfo(streamTaskService.getTask(jobId,version)); + jobDetailsVO.setLinkisJobInfo(streamTaskService.getTaskJobInfo(jobId,version)); jobDetailsVO.setDataNumber(dataNumberDTOS); jobDetailsVO.setLoadCondition(loadConditionDTOs); jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); @@ -238,6 +242,102 @@ public Message executeHistoryJob(HttpServletRequest req, return Message.ok().data("details", details); } + @RequestMapping(path = "/addTask", method = RequestMethod.GET) + public Message addTask(HttpServletRequest req, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "appUrl") String appUrl) { + String username = SecurityFilter.getLoginUsername(req); + LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + List streamJobs = streamJobService.getJobByName(jobName); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to add new tasks."); + } + if (!streamJobService.hasPermission(streamJobs.get(0), username) && + !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { + return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); + } + // 如果存在正在运行的，先将其停止掉 + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTaskService.updateTask(streamTask); + } else { + // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo + // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 + PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, 0, null); + Optional copyJob = jobList.getList().stream().min((job1, job2) -> { + if (job1.getStatus() > 0) { + return 0; + } else { + return 1; + } + }); + if(!copyJob.isPresent()) { + return Message.error("If no Flink Job has submitted, the register to Streamis cannot be succeeded."); + } + StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJob.get().getId()); + LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", + jobName, copyJob.get().getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); + streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setLinkisJobId(copyTask.getLinkisJobId()); + streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); + } + streamTask.setStartTime(new Date()); + streamTask.setLastUpdateTime(new Date()); + FlinkJobInfo flinkJobInfo; + try { + flinkJobInfo = DWSHttpClient.jacksonJson().readValue(streamTask.getLinkisJobInfo(), FlinkJobInfo.class); + } catch (JsonProcessingException e) { + LOG.error("Job {} deserialize the jobInfo from history Job failed!", jobName, e); + return Message.error("Deserialize the jobInfo from history Job failed!"); + } + flinkJobInfo.setApplicationId(appId); + flinkJobInfo.setApplicationUrl(appUrl); + flinkJobInfo.setName(jobName); + flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); + StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); + streamTaskService.updateTask(streamTask); + LOG.info("Streamis Job {} has added a new task successfully.", jobName); + return Message.ok(); + } + + @RequestMapping(path = "/stopTask", method = RequestMethod.GET) + public Message stopTask(HttpServletRequest req, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "appUrl") String appUrl) { + String username = SecurityFilter.getLoginUsername(req); + LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + List streamJobs = streamJobService.getJobByName(jobName); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to add new tasks."); + } + if (!streamJobService.hasPermission(streamJobs.get(0), username) && + !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { + return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); + } + // 如果存在正在运行的，将其停止掉 + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTaskService.updateTask(streamTask); + } else { + LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", jobName); + } + return Message.ok(); + } + @RequestMapping(path = "/progress", method = RequestMethod.GET) public Message progressJob(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, @RequestParam(value = "version", required = false) String version) throws IOException, JobException { From 3afd19ccec925e2849bcc4fa2c2c608ddafc0731 Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 19 Oct 2022 02:38:31 +0800 Subject: [PATCH 004/219] Use the "\u0001" instead of "\0x001" to act as blank placeholder. --- .../linkis/job/manager/SimpleFlinkJobLaunchManager.scala | 1 - .../streamis/jobmanager/manager/conf/JobConf.scala | 2 ++ .../streamis/jobmanager/manager/utils/JobUtils.scala | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala index 5e69766fe..8b4308a2b 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala @@ -69,7 +69,6 @@ class SimpleFlinkJobLaunchManager extends FlinkJobLaunchManager { jobInfo.setCompletedMsg(message) } jobInfo.setResources(nodeInfo.get("nodeResource").asInstanceOf[util.Map[String, Object]]) - jobInfo.setStatus("failed") // Set job state info into // Option(jobState).foreach(state => { // val stateInfo = new JobStateInfo diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 89cab6f92..7da10b5f5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -27,6 +27,8 @@ object JobConf { val STREAMIS_JOB_MONITOR_ENABLE: CommonVars[Boolean] = CommonVars("wds.streamis.job.monitor.enable", true) + val STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER: CommonVars[String] = CommonVars("wds.streamis.job.param.blank.placeholder", "\u0001") + val FLINK_JOB_STATUS_NOT_STARTED: CommonVars[Int] = CommonVars("wds.streamis.job.status.not-started", 0,"Not Started") val FLINK_JOB_STATUS_COMPLETED: CommonVars[Int] = CommonVars("wds.streamis.job.status.completed", 1,"Completed") diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala index 485f9c8ce..75a62b53d 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala @@ -1,5 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.utils +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf + import java.util import scala.collection.JavaConverters.{asScalaSetConverter, mapAsScalaMapConverter} @@ -12,7 +14,7 @@ object JobUtils { for (paramEntry <- params.entrySet().asScala){ val value = paramEntry.getValue value match { - case str: String => paramEntry.setValue(str.replace(" ", "\\0x001")) + case str: String => paramEntry.setValue(str.replace(" ", JobConf.STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER.getValue)) case _ => } } From 9288a1cba6c7aba6c24023f2dffebbb2f49ffc5f Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 19 Oct 2022 02:42:57 +0800 Subject: [PATCH 005/219] SendBuffer and BucketConfig. --- .../collector/sender/AbstractSendBuffer.java | 107 ++++++++++++++++++ .../collector/sender/ImmutableSendBuffer.java | 71 ++++++++++++ .../log/collector/sender/SendBuffer.java | 77 +++++++++++++ .../streamis-job-log-server/pom.xml | 4 + .../exception/StreamJobLogException.java | 29 +++++ .../storage/bucket/JobLogBucketConfig.java | 103 +++++++++++++++-- 6 files changed, 379 insertions(+), 12 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java new file mode 100644 index 000000000..2f778b231 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java @@ -0,0 +1,107 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +/** + * Abstract sender buffer; + * non-blocking and reduces out-of-bounds exceptions + */ +public abstract class AbstractSendBuffer implements SendBuffer{ + + protected enum Flag{ + WRITE_MODE, READ_MODE + } + + /** + * Access flag + */ + private Flag accessFlag = Flag.WRITE_MODE; + + private int position = 0; + private int limit; + /** + * The capacity is mutable + */ + protected int capacity; + + + public AbstractSendBuffer(int capacity){ + this.capacity = capacity; + limit(this.capacity); + } + + public AbstractSendBuffer(){ + this(Integer.MAX_VALUE); + } + @Override + public int capacity() { + return this.capacity; + } + + @Override + public int remaining() { + int rem = this.limit - this.position; + return Math.max(rem, 0); + } + + @Override + public void flip() { + this.limit = this.position; + this.position = 0; + this.accessFlag = Flag.READ_MODE; + } + + @Override + public void clear() { + limit(this.capacity); + this.position = 0; + this.accessFlag = Flag.WRITE_MODE; + clearBuf(); + } + + /** + * Change the limit value + * @param newLimit new limit + */ + final void limit(int newLimit){ + if (newLimit > this.capacity || (newLimit < 0)){ + throw new IllegalArgumentException("Set the illegal limit value: " + newLimit + " in send buffer, [capacity: " + this.capacity + "]"); + } + this.limit = newLimit; + if (this.position > newLimit){ + this.position = newLimit; + } + } + + /** + * Inc the position with offset + * @param offset offset value + * @param accessFlag access flag + * @return the current position value + */ + final int nextPosition(int offset, Flag accessFlag){ + if (this.accessFlag != accessFlag){ + throw new IllegalStateException("Illegal access flag [" + accessFlag + "] for send buffer"); + } + int p = position; + // Reach the limit, return -1 value + if (p >= limit){ + return -1; + } + if (p + offset > limit){ + this.position = limit; + } + return p; + } + + /** + * + * @return the current position + */ + final int position(){ + return this.position; + } + + /** + * Do the actual clear + */ + protected abstract void clearBuf(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java new file mode 100644 index 000000000..3587079fa --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import java.util.Arrays; +import java.util.function.Function; + +/** + * Immutable send buffer (use array) + */ +public class ImmutableSendBuffer extends AbstractSendBuffer{ + + /** + * Buffer object array + */ + private final Object[] buf; + + public ImmutableSendBuffer(int capacity) { + super(capacity); + buf = new Object[capacity]; + } + + @Override + protected void clearBuf() { + // Release the memory occupied + Arrays.fill(buf, null); + } + + @Override + public void capacity(String newCapacity) { + throw new IllegalArgumentException("Unsupported to scale-in/scale-up the send buffer"); + } + + @Override + public int writeBuf(E[] elements, int srcIndex, int length) { + if (srcIndex < elements.length){ + int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); + if (startPos >= 0){ + int writes = position() - startPos; + for (int i = srcIndex; i < writes; i ++){ + buf[startPos++] = elements[i]; + } + return writes; + } + } + return -1; + } + + @Override + public int readBuf(E[] elements, int srcIndex, int length) { + return 0; + } + + @Override + public int writeBuf(E element) { + int startPos = nextPosition(1, Flag.WRITE_MODE); + if (startPos >= 0){ + buf[startPos] = element; + return 1; + } + return -1; + } + + @Override + public E readBuf() { + return null; + } + + @Override + public SendBuffer compact(Function dropAble) { + return null; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java new file mode 100644 index 000000000..e8bee928b --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import java.util.function.Function; + +/** + * Buffer for Rpc sender + * @param buffer element + */ +public interface SendBuffer { + + /** + * Capacity + * @return int + */ + int capacity(); + + /** + * Scale-up or scale-in + * @param newCapacity new capacity + */ + void capacity(String newCapacity); + /** + * Remain size + * (remain space for writing or remain elements for reading) + * @return int + */ + int remaining(); + + /** + * transient between write-mode and read-mode + */ + void flip(); + + /** + * Clear to reuse the buffer + */ + void clear(); + /** + * Write buffer element + * @param element element + * @return if succeed + */ + int writeBuf(E element); + + /** + * Write buffer element array + * @param elements elements + * @param srcIndex the src index in elements + * @param length the length to read + * @return write num + */ + int writeBuf(E[] elements, int srcIndex, int length); + + /** + * Read buffer element + * @return element + */ + E readBuf(); + + /** + * Read buffer element array + * @param elements elements + * @param srcIndex the src index in elements + * @param length the length to write + * @return read num + */ + int readBuf(E[] elements, int srcIndex, int length); + + /** + * Compact the buffer, avoid the useless elements + * @param dropAble drop function + * @return send buffer + */ + SendBuffer compact(Function dropAble); + + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 69936a7b4..7f2a5bc10 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -22,5 +22,9 @@ streamis-job-log-common 0.2.0 + + org.apache.linkis + linkis-module + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java new file mode 100644 index 000000000..56edc2dd3 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.exception; + +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.ExceptionLevel; +import org.apache.linkis.common.exception.LinkisRuntimeException; + +/** + * Stream job log exception + */ +public class StreamJobLogException extends ErrorException { + public StreamJobLogException(int errCode, String desc) { + super(errCode, desc); + } + public StreamJobLogException(int errCode, String desc, Throwable t){ + super(errCode, desc); + + } + public static class Runtime extends LinkisRuntimeException{ + + public Runtime(int errCode, String desc) { + super(errCode, desc); + } + + @Override + public ExceptionLevel getLevel() { + return ExceptionLevel.ERROR; + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java index 85a5c2d2c..2dc82e40f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java @@ -1,5 +1,8 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; +import org.apache.linkis.common.conf.CommonVars; + import java.util.HashMap; import java.util.Map; @@ -8,25 +11,73 @@ */ public class JobLogBucketConfig { + @SuppressWarnings("unchecked") + public JobLogBucketConfig(){ + try { + Class defaultBucketClass = Class.forName(Define.JOB_LOG_BUCKET_CLASS.getValue()); + if (JobLogBucket.class.isAssignableFrom(defaultBucketClass)){ + this.bucketClass = (Class) defaultBucketClass; + } + } catch (ClassNotFoundException e) { +// throw new StreamJobLogException.Runtime(-1, "", e); + } + } + /** * Bucket class */ private Class bucketClass; + /** + * Root path for bucket + */ + private String bucketRootPath; + /** * Attribute */ protected Map attributes = new HashMap<>(); /** - * Max size of bucket part + * Max size of bucket active part */ - private int maxBucketPartSize; + private int maxBucketActivePartSize; + + /** + * Max number of bucket part + */ + private int maxBucketPartNum; + + /** + * The compress format used for bucket parts + */ + private String bucketPartCompress; + + /** + * Max hold time in minutes for bucket part + */ + private long bucketPartHoldTimeInMin; /** * Layout pattern */ - private String LayOutPattern; + private String LogLayOutPattern = "%msg%n"; + + public Class getBucketClass() { + return bucketClass; + } + + public void setBucketClass(Class bucketClass) { + this.bucketClass = bucketClass; + } + + public String getBucketRootPath() { + return bucketRootPath; + } + + public void setBucketRootPath(String bucketRootPath) { + this.bucketRootPath = bucketRootPath; + } public Map getAttributes() { return attributes; @@ -36,23 +87,51 @@ public void setAttributes(Map attributes) { this.attributes = attributes; } - public int getMaxBucketPartSize() { - return maxBucketPartSize; + public int getMaxBucketActivePartSize() { + return maxBucketActivePartSize; } - public void setMaxBucketPartSize(int maxBucketPartSize) { - this.maxBucketPartSize = maxBucketPartSize; + public void setMaxBucketActivePartSize(int maxBucketActivePartSize) { + this.maxBucketActivePartSize = maxBucketActivePartSize; } - public String getLayOutPattern() { - return LayOutPattern; + public int getMaxBucketPartNum() { + return maxBucketPartNum; } - public void setLayOutPattern(String layOutPattern) { - LayOutPattern = layOutPattern; + public void setMaxBucketPartNum(int maxBucketPartNum) { + this.maxBucketPartNum = maxBucketPartNum; } - public static final class Define{ + public String getBucketPartCompress() { + return bucketPartCompress; + } + + public void setBucketPartCompress(String bucketPartCompress) { + this.bucketPartCompress = bucketPartCompress; + } + + public long getBucketPartHoldTimeInMin() { + return bucketPartHoldTimeInMin; + } + + public void setBucketPartHoldTimeInMin(long bucketPartHoldTimeInMin) { + this.bucketPartHoldTimeInMin = bucketPartHoldTimeInMin; + } + + public String getLogLayOutPattern() { + return LogLayOutPattern; + } + public void setLogLayOutPattern(String logLayOutPattern) { + LogLayOutPattern = logLayOutPattern; + } + + + public static final class Define{ + /** + * Default bucket class + */ + public static final CommonVars JOB_LOG_BUCKET_CLASS = CommonVars.apply("wds.streamis.job.log.bucket.class", "com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.Log4j2JobLogBucket"); } } From a7183e99ce6624cb8a786dbdab8e5f205a02d66f Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 19 Oct 2022 21:35:34 +0800 Subject: [PATCH 006/219] Add the new feature for adding task and stop task for existed jobs. --- .../jobmanager/manager/conf/JobConf.scala | 3 + .../service/DefaultStreamJobService.scala | 5 +- .../parser/SparkJarJobContentParser.scala | 78 +++++++++++++++++++ .../jobmanager/restful/api/JobRestfulApi.java | 71 +++++++++++------ 4 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 89cab6f92..5f33fff2f 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -82,4 +82,7 @@ object JobConf { val TASK_SUBMIT_TIME_MAX: CommonVars[TimeType] = CommonVars("wds.streamis.task.submit.time.max", new TimeType("5m")) + val SUPPORTED_JOB_TYPES: CommonVars[String] = CommonVars("wds.streamis.supported.job.types", "flink.jar,flink.sql,spark.jar") + + val SUPPORTED_MANAGEMENT_JOB_TYPES: CommonVars[String] = CommonVars("wds.streamis.management.supported.job.types", "flink.jar,flink.sql") } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index f12878433..e0c270d99 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -24,8 +24,8 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobCon import com.webank.wedatasphere.streamis.jobmanager.manager.alert.AlertLevel import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamAlertMapper, StreamJobMapper, StreamTaskMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{QueryJobListVo, TaskCoreNumVo, VersionDetailVo} import com.webank.wedatasphere.streamis.jobmanager.manager.entity._ +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{QueryJobListVo, TaskCoreNumVo, VersionDetailVo} import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobCreateErrorException, JobFetchErrorException} import com.webank.wedatasphere.streamis.jobmanager.manager.transform.JobContentParser import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent @@ -128,6 +128,9 @@ class DefaultStreamJobService extends StreamJobService with Logging { override def createStreamJob(metaJsonInfo: MetaJsonInfo, userName: String): StreamJobVersion = { if(StringUtils.isBlank(metaJsonInfo.getJobType)) throw new JobCreateErrorException(30030, s"jobType is needed.") + else if(!JobConf.SUPPORTED_JOB_TYPES.getValue.contains(metaJsonInfo.getJobType)) { + throw new JobCreateErrorException(30030, s"jobType ${metaJsonInfo.getJobType} is not supported.") + } if(metaJsonInfo.getJobContent == null || metaJsonInfo.getJobContent.isEmpty) throw new JobCreateErrorException(30030, s"jobContent is needed.") val job = streamJobMapper.getCurrentJob(metaJsonInfo.getProjectName, metaJsonInfo.getJobName) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala new file mode 100644 index 000000000..d730753c2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamJobVersion, StreamJobVersionFiles, StreamisFile} +import com.webank.wedatasphere.streamis.jobmanager.manager.exception.JobExecuteErrorException +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.{StreamisJarTransformJobContent, StreamisTransformJobContent} +import org.apache.commons.lang.StringUtils +import org.apache.linkis.common.utils.JsonUtils +import org.apache.linkis.manager.label.entity.engine.RunType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.springframework.stereotype.Component + +import scala.collection.JavaConverters._ + +/** + * + * @date 2022-10-19 + * @author enjoyyin + * @since 0.5.0 + */ +@Component +class SparkJarJobContentParser extends AbstractJobContentParser { + + override val jobType: String = "spark.jar" + override val runType: RunType = RunType.JAR + + override def parseTo(job: StreamJob, jobVersion: StreamJobVersion): StreamisTransformJobContent = { + val createFile: String => StreamisFile = fileName => { + val file = new StreamJobVersionFiles() + file.setFileName(fileName) + file.setCreateBy(job.getCreateBy) + file.setCreateTime(job.getCreateTime) + file.setJobId(job.getId) + file.setJobVersionId(jobVersion.getId) + file.setVersion(jobVersion.getVersion) + file.setStorePath("") + file.setStoreType("") + file + } + val transformJobContent = new StreamisJarTransformJobContent + val jobContent = JsonUtils.jackson.readValue(jobVersion.getJobContent, classOf[util.Map[String, Object]]) + jobContent.get("main.class.jar") match { + case mainClassJar: String => + transformJobContent.setMainClassJar(createFile(mainClassJar)) + case _ => throw new JobExecuteErrorException(30500, "main.class.jar is needed.") + } + jobContent.get("main.class") match { + case mainClass: String => + transformJobContent.setMainClass(mainClass) + case _ => throw new JobExecuteErrorException(30500, "main.class is needed.") + } + jobContent.get("args") match { + case args: util.List[String] => + transformJobContent.setArgs(args) + case _ => + } + jobContent.get("hdfs.jars") match { + case hdfsJars: util.List[String] => + transformJobContent.setHdfsJars(hdfsJars) + case _ => + } + jobContent.get("dependency.jars") match { + case dependencyJars: util.List[String] => + val parsedDependencyJars = dependencyJars.asScala.filter(StringUtils.isNotBlank).map(createFile).asJava + transformJobContent.setDependencyJars(parsedDependencyJars) + case _ => + } + jobContent.get("resources") match { + case resources: util.List[String] => + val parsedResources = resources.asScala.filter(StringUtils.isNotBlank).map(createFile).asJava + transformJobContent.setResources(parsedResources) + case _ => + } + transformJobContent + } + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index c6c9b5a97..bba1dc579 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -51,6 +51,7 @@ import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; @RequestMapping(path = "/streamis/streamJobManager/job") @RestController @@ -77,7 +78,7 @@ public Message getJobList(HttpServletRequest req, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "jobName", required = false) String jobName, @RequestParam(value = "jobStatus", required = false) Integer jobStatus, - @RequestParam(value = "jobCreator", required = false) String jobCreator) throws JobException { + @RequestParam(value = "jobCreator", required = false) String jobCreator) { String username = SecurityFilter.getLoginUsername(req); if(StringUtils.isBlank(projectName)){ return Message.error("Project name cannot be empty(项目名不能为空，请指定)"); @@ -100,7 +101,7 @@ public Message getJobList(HttpServletRequest req, } @RequestMapping(path = "/createOrUpdate", method = RequestMethod.POST) - public Message createOrUpdate(HttpServletRequest req, @Validated @RequestBody MetaJsonInfo metaJsonInfo) throws Exception { + public Message createOrUpdate(HttpServletRequest req, @Validated @RequestBody MetaJsonInfo metaJsonInfo) { String username = SecurityFilter.getLoginUsername(req); String projectName = metaJsonInfo.getProjectName(); if (StringUtils.isBlank(projectName)){ @@ -142,6 +143,11 @@ public Message executeJob(HttpServletRequest req, @RequestBody Map jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, 0, null); - Optional copyJob = jobList.getList().stream().min((job1, job2) -> { - if (job1.getStatus() > 0) { - return 0; + List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark") && job.getStatus() > 0) + .collect(Collectors.toList()); + if(copyJobs.isEmpty()) { + return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + } + int index = 0; + while(streamTask == null && index < copyJobs.size()) { + StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); + if(copyTask == null) { + index ++; } else { - return 1; + LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", + jobName, copyJobs.get(index).getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); + streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setLinkisJobId(copyTask.getLinkisJobId()); + streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); } - }); - if(!copyJob.isPresent()) { - return Message.error("If no Flink Job has submitted, the register to Streamis cannot be succeeded."); } - StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJob.get().getId()); - LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", - jobName, copyJob.get().getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); - streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); - streamTask.setLinkisJobId(copyTask.getLinkisJobId()); - streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); + if(streamTask == null) { + return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + } } streamTask.setStartTime(new Date()); streamTask.setLastUpdateTime(new Date()); @@ -320,11 +336,11 @@ public Message stopTask(HttpServletRequest req, } else if(streamJobs.size() > 1) { return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to add new tasks."); + return Message.error("Only spark.jar Job support to stop task."); } if (!streamJobService.hasPermission(streamJobs.get(0), username) && !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { - return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); + return Message.error("Have no permission to stop task for StreamJob [" + jobName + "]."); } // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); @@ -340,12 +356,17 @@ public Message stopTask(HttpServletRequest req, @RequestMapping(path = "/progress", method = RequestMethod.GET) public Message progressJob(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, - @RequestParam(value = "version", required = false) String version) throws IOException, JobException { + @RequestParam(value = "version", required = false) String version) throws JobException { String username = SecurityFilter.getLoginUsername(req); if (jobId == null) { throw JobExceptionManager.createException(30301, "jobId"); } StreamJob streamJob = this.streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { + return Message.error("Job " + streamJob.getName() + " is not supported to get progress."); + } if (!streamJobService.hasPermission(streamJob, username) && !this.privilegeService.hasAccessPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to view the progress of StreamJob [" + jobId + "]"); @@ -371,7 +392,6 @@ public Message uploadDetailsJob(HttpServletRequest req, @RequestParam(value = "j public Message getAlert(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, @RequestParam(value = "version", required = false) String version) { String username = SecurityFilter.getLoginUsername(req); - return Message.ok().data("list", streamJobService.getAlert(username, jobId, version)); } @@ -436,6 +456,11 @@ public Message snapshot(@PathVariable("jobId")Long jobId, HttpServletRequest req try{ String username = SecurityFilter.getLoginUsername(request); StreamJob streamJob = this.streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { + return Message.error("Job " + streamJob.getName() + " is not supported to do snapshot."); + } if (!streamJobService.hasPermission(streamJob, username) && !this.privilegeService.hasEditPrivilege(request, streamJob.getProjectName())){ return Message.error("Have no permission to do snapshot for StreamJob [" + jobId + "]"); From ba4189dd979eb12054d9e4732e250a9801a1c55c Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 20 Oct 2022 03:11:32 +0800 Subject: [PATCH 007/219] Complete the log collector (80%). --- .../streamis-job-log-collector/pom.xml | 7 + .../collector/StreamisLogAppenderConfig.java | 4 + .../log/collector/StreamisRpcLogAppender.java | 5 + .../log/collector/cache/LogCache.java | 43 ++ .../sender/AbstractRpcLogSender.java | 428 ++++++++++++++++++ .../log/collector/sender/RpcLogSender.java | 33 ++ .../log/collector/sender/RpcSenderConfig.java | 34 ++ .../log/collector/sender/SendLogCache.java | 20 + .../sender/SendLogCacheConsumer.java | 99 ++++ .../sender/SendLogExceptionStrategy.java | 62 +++ .../sender/StreamisRpcLogSender.java | 6 + .../sender/{ => buf}/AbstractSendBuffer.java | 18 +- .../sender/{ => buf}/ImmutableSendBuffer.java | 2 +- .../sender/{ => buf}/SendBuffer.java | 19 +- .../sender/http/AbstractHttpLogSender.java | 24 + 15 files changed, 800 insertions(+), 4 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/{ => buf}/AbstractSendBuffer.java (89%) rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/{ => buf}/ImmutableSendBuffer.java (99%) rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/{ => buf}/SendBuffer.java (83%) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index aa20062f4..9549d3439 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -16,4 +16,11 @@ 8 + + + com.webank.wedatasphere.streamis + streamis-job-log-common + 0.2.0 + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java new file mode 100644 index 000000000..5c0ff730d --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +public class StreamisLogAppenderConfig { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java new file mode 100644 index 000000000..d326e6e04 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +public class StreamisRpcLogAppender { + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java new file mode 100644 index 000000000..f11556cc8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.cache; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Log cache + * @param element + */ +public interface LogCache { + + /** + * Cache log + * @param logElement log element + */ + void cacheLog(E logElement) throws InterruptedException; + + /** + * Drain log elements into collection + * @param elements elements + * @param maxElements max elements size + * @return count + */ + int drainLogsTo(List elements, int maxElements); + + /** + * Take log element + * @return log element + */ + E takeLog(long timeout, TimeUnit unit) throws InterruptedException; + + /** + * If the cache is full + * @return + */ + boolean isCacheable(); + /** + * Release the resource + */ + void destroy(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java new file mode 100644 index 000000000..3c010878b --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -0,0 +1,428 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.ImmutableSendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Abstract rpc log sender + * @param + * @param + */ +public abstract class AbstractRpcLogSender implements RpcLogSender{ + + /** + * Size of log cache + */ + int cacheSize; + + /** + * The buffer size of sender + */ + int sendBufSize; + + /** + * Max thread num of send + */ + int maxCacheConsume; + /** + * Connect config + */ + protected RpcSenderConfig rpcSenderConfig; + + /** + * Rpc log context + */ + private volatile RpcLogContext rpcLogContext; + + + public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize){ + this(rpcSenderConfig, cacheSize, sendBufSize, Integer.MAX_VALUE); + } + + public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize, int maxCacheConsume){ + this.rpcSenderConfig = rpcSenderConfig; + if (sendBufSize > cacheSize) { + throw new IllegalArgumentException("Size of send buffer is larger than cache size"); + } + this.cacheSize = cacheSize; + this.sendBufSize = sendBufSize; + this.maxCacheConsume = maxCacheConsume; + } + + @Override + public LogCache getOrCreateLogCache() { + getOrCreateRpcLogContext(); + return null; + } + + @Override + public void sendLog(T log) { + // Just send it into log cache + try { + getOrCreateLogCache().cacheLog(log); + } catch (InterruptedException e) { + // Exception handler + } + } + + @Override + public void syncSendLog(T log) { + + } + + @Override + public void close() { + + } + + /** + * Aggregate send buffer for sending + * @param sendBuffer send buffer + * @return E aggregated entity + */ + protected abstract E aggregateBuffer(SendBuffer sendBuffer); + + /** + * Sending operation + * @param aggregatedEntity agg entity + * @param rpcSenderConfig rpc sender config + */ + protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig); + + /** + * Send log exception strategy + * @return exception strategy + */ + protected abstract SendLogExceptionStrategy getSendLogExceptionStrategy(); + + protected RpcLogContext getOrCreateRpcLogContext(){ + if (null == this.rpcLogContext){ + synchronized (this){ + if (null == this.rpcLogContext){ + SendLogCache logCache = new QueuedSendLogCache(this.cacheSize, false); + this.rpcLogContext = new RpcLogContext(logCache); + // Start cache consumer + this.rpcLogContext.startCacheConsumer(); + } + } + + } + return this.rpcLogContext; + } + + private class RpcLogContext{ + + private static final String RPC_LOG_CACHE_CONSUMER = "RpcLog-Cache-Consumer-Thread-"; + /** + * Send log cache + */ + private final SendLogCache logCache; + + /** + * Consume pool + */ + private final ThreadPoolExecutor consumePool; + + /** + * Count of the consumers + */ + private int consumers = 0; + + public RpcLogContext(SendLogCache logCache){ + this.logCache = logCache; + this.consumePool = new ThreadPoolExecutor(0, maxCacheConsume, + 60L, TimeUnit.SECONDS, + new SynchronousQueue<>(), new ThreadFactory() { + private final ThreadGroup group = Thread.currentThread().getThreadGroup(); + private final AtomicInteger threadNum = new AtomicInteger(1); + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, RPC_LOG_CACHE_CONSUMER + + threadNum.getAndIncrement(), 0); + if (t.isDaemon()) { + t.setDaemon(false); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } + }); + } + + public synchronized void startCacheConsumer(){ + if (consumers >= maxCacheConsume){ + throw new IllegalStateException("Over the limit number of cache consumers: [" + maxCacheConsume + "]"); + } + String id = UUID.randomUUID().toString(); + SendBuffer sendBuffer = new ImmutableSendBuffer<>(sendBufSize); + this.consumePool.submit(new SendLogCacheConsumer(id, logCache, sendBuffer, rpcSenderConfig) { + @Override + protected void onFlushAndSend(SendBuffer sendBuffer) { + // First to aggregate the buffer + E aggEntity = aggregateBuffer(sendBuffer); + Optional.ofNullable(getSendLogExceptionStrategy()).ifPresent( + strategy -> strategy.doSend(() -> { + doSend(aggEntity, rpcSenderConfig); + return null; + }, sendBuffer)); + } + }); + this.consumers ++; + } + + public SendLogCache getLogCache(){ + return this.logCache; + } + + public void destroyCacheConsumer(){ + + } + } + /** + * Act as ArrayBlockingQueue (jdk 1.8) + */ + private class QueuedSendLogCache implements SendLogCache{ + + // Queued items + final Object[] items; + + // Take index + int takeIndex; + + // Put index + int putIndex; + + // Count + int count; + + // Reentrant lock + final ReentrantLock lock; + + // Condition for waiting takes + private final Condition notEmpty; + + // Condition for waiting puts(cacheLog) + private final Condition notFull; + + public QueuedSendLogCache(int capacity, boolean fair) { + this.items = new Object[capacity]; + lock = new ReentrantLock(fair); + this.notEmpty = lock.newCondition(); + this.notFull = lock.newCondition(); + } + + @Override + public void cacheLog(T logElement) throws InterruptedException { + // Skip the null element + if (Objects.nonNull(logElement)){ + final ReentrantLock lock = this.lock; + lock.lockInterruptibly(); + try{ + while (count == items.length){ + notFull.await(); + } + enqueue(logElement); + }finally{ + lock.unlock(); + } + } + } + + @Override + public int drainLogsTo(List elements, int maxElements) { + if (Objects.nonNull(elements) && maxElements > 0){ + final Object[] items = this.items; + final ReentrantLock lock = this.lock; + lock.lock(); + try{ + int n = Math.min(maxElements, count); + int take = takeIndex; + int i = 0; + try { + while (i < n){ + @SuppressWarnings("unchecked") + T x = (T) items[take]; + elements.add(x); + items[take] = null; + if (++ take == items.length) + take = 0; + i++; + } + return n; + }finally { + restoreInvariants(i, take, false); + } + } finally { + lock.unlock(); + } + } + return 0; + } + + // Equal to the poll method in ArrayBlockingQuue + @Override + public T takeLog(long timeout, TimeUnit unit) throws InterruptedException { + long nanos = unit.toNanos(timeout); + final ReentrantLock lock = this.lock; + T element; + lock.lockInterruptibly(); + try{ + while (count == 0){ + if (nanos < 0){ + return null; + } + nanos = notEmpty.awaitNanos(nanos); + } + element = dequeue(); + } finally { + lock.unlock(); + } + return element; + } + + @Override + public boolean isCacheable() { + final ReentrantLock lock = this.lock; + lock.lock(); + try { + return count < items.length; + }finally { + lock.unlock(); + } + } + + // The same as the clear() method, + @Override + public void destroy() { + final Object[] items = this.items; + final ReentrantLock lock = this.lock; + lock.lock(); + try { + int k = count; + if (k > 0) { + final int putIndex = this.putIndex; + int i = takeIndex; + do { + items[i] = null; + if (++i == items.length) + i = 0; + } while (i != putIndex); + takeIndex = putIndex; + count = 0; + for (; k > 0 && lock.hasWaiters(notFull); k--) + notFull.signal(); + } + } finally { + lock.unlock(); + } + } + + /** + * Drain the elements into send buffer + * @param sendBuffer send buffer + * @param maxElements max element size + * @return int + */ + @Override + public int drainLogsTo(SendBuffer sendBuffer, int maxElements) { + if (Objects.nonNull(sendBuffer) && maxElements > 0){ + final Object[] items = this.items; + final ReentrantLock lock = this.lock; + lock.lock(); + try{ + int n = Math.min(maxElements, count); + int take = takeIndex; + int i = 0; + int send; + try { + while (n > 0) { + int len = items.length - take; + int send0 = Math.min(n, len); + // Copy the array element to buffer directly + send = sendBuf(sendBuffer, this.items, take, send0); + n -= send; + if ((take = take + send) >= items.length) { + take = 0; + } + i += send; + if (send < send0 || send <= 0) { + break; + } + } + return i; + } finally { + if (i > 0){ + restoreInvariants(i, take, true); + } + } + }finally { + lock.unlock(); + } + } + return 0; + } + + @SuppressWarnings("unchecked") + private int sendBuf(SendBuffer sendBuffer, Object[] items, int takeIndex, int len){ + int send = sendBuffer.writeBuf((T[]) items, takeIndex, len); + if (send < len){ + // Buffer full exception + } + // Allow data loss + return send; + } + + private void restoreInvariants(int i, int take, boolean clearItems){ + this.count -= i; + if (clearItems){ + int index = this.takeIndex; + for (; i > 0; i --){ + this.items[index] = null; + if (++index == items.length){ + index = 0; + } + } + //At last index equals take + } + this.takeIndex = take; + for (; i > 0 && lock.hasWaiters(notFull); i--){ + notFull.signal(); + } + } + // Inserts element at current put position, advances, and signals. Call only when holding lock. + private void enqueue(T element){ + this.items[putIndex] = element; + if (++putIndex >= items.length){ + putIndex = 0; + } + count ++; + notEmpty.signal(); + } + + // Extracts element at current take position, advances, and signals. Call only when holding lock. + private T dequeue(){ + @SuppressWarnings("unchecked") + T element = (T)this.items[takeIndex]; + this.items[takeIndex] = null; + if ( ++ takeIndex == items.length){ + this.takeIndex = 0; + } + count --; + // Not need to support iterator + notFull.signal(); + return element; + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java new file mode 100644 index 000000000..25f512643 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +/** + * Rpc Log sender + */ +public interface RpcLogSender { + + /** + * Produce log cache + * @return log cache + */ + LogCache getOrCreateLogCache(); + + /** + * Send log (async) + * @param log log element + */ + void sendLog(T log); + + /** + * Send log (sync) + * @param log log element + */ + void syncSendLog(T log); + + /** + * Close sender + */ + void close(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java new file mode 100644 index 000000000..a0d3a77e5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +/** + * Rpc sender configuration + */ +public class RpcSenderConfig { + + /** + * Send address + */ + private String sendAddress; + /** + * Size of send cache + */ + private int sendCacheSize = 150; + + /** + * Size of send buffer + */ + private int sendBufferSize = 50; + + /** + * Expire time of send buffer + */ + private long sendBufferExpireTimeInSec = -1; + + public long getSendBufferExpireTimeInSec() { + return sendBufferExpireTimeInSec; + } + + public void setSendBufferExpireTimeInSec(long sendBufferExpireTimeInSec) { + this.sendBufferExpireTimeInSec = sendBufferExpireTimeInSec; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java new file mode 100644 index 000000000..200c573d9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +/** + * Send log cache + * @param + */ +public interface SendLogCache extends LogCache { + + /** + * Drain the logs into send buffer + * @param sendBuffer send buffer + * @param maxElements max element size + * @return count + */ + int drainLogsTo(SendBuffer sendBuffer, int maxElements);; +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java new file mode 100644 index 000000000..f0d8f5866 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.concurrent.TimeUnit; + +/** + * Send log consumer + * Consume the log elements from cache and put into send buffer + * @param + */ +public abstract class SendLogCacheConsumer implements Runnable{ + + private boolean isTerminated = false; + + /** + * Buffer expire time in milliseconds + */ + private final long bufferExpireTimeInMills; + /** + * Send log cache + */ + private final SendLogCache cache; + + /** + * Send buffer + */ + private final SendBuffer sendBuffer; + + private final String id; + public SendLogCacheConsumer(String id, SendLogCache cache, + SendBuffer sendBuffer, + RpcSenderConfig rpcSenderConfig){ + this.id = id; + this.cache = cache; + this.sendBuffer = sendBuffer; + this.bufferExpireTimeInMills = rpcSenderConfig.getSendBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS + .toMillis(rpcSenderConfig.getSendBufferExpireTimeInSec()) : -1; + + } + + @Override + public void run() { + int remain; + long expireTimeInMills = requireNewFlushTime(); + while (!this.isTerminated) { + try { + if ((expireTimeInMills > 0 && expireTimeInMills >= System.currentTimeMillis()) + || (remain = this.sendBuffer.remaining()) <= 0) { + // Transient to the read mode + sendBuffer.flip(); + onFlushAndSend(sendBuffer); + expireTimeInMills = requireNewFlushTime(); + if (sendBuffer.isReadMode()) { + // Clear the buffer and transient to the write mode, otherwise continue writing + sendBuffer.clear(); + } + remain = this.sendBuffer.remaining(); + } + if (remain > 0) { + int inBuf = this.cache.drainLogsTo(sendBuffer, remain); + if (inBuf < remain) { + // Means that the cache is empty, take and wait the log element + long waitTime = expireTimeInMills - System.currentTimeMillis(); + if (waitTime > 0) { + T logElement = this.cache.takeLog(waitTime, TimeUnit.MILLISECONDS); + if (null != logElement) { + sendBuffer.writeBuf(logElement); + } + } + } + } + } catch (Throwable e){ + if (this.isTerminated && e instanceof InterruptedException){ + return; + } else { + System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); + // For the unknown exception clear the cache + sendBuffer.clear(); + expireTimeInMills = requireNewFlushTime(); + } + } + } + } + + public void shutdown(){ + this.isTerminated = true; + } + + private long requireNewFlushTime(){ + return bufferExpireTimeInMills > 0 ? System.currentTimeMillis() + bufferExpireTimeInMills : -1; + } + /** + * When the buffer is full or reach the idle time, invoke the method + * @param sendBuffer send buffer + */ + protected abstract void onFlushAndSend(SendBuffer sendBuffer); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java new file mode 100644 index 000000000..d3c2fafd9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.Objects; +import java.util.concurrent.Callable; + +/** + * Strategy control the action on exception + */ +public abstract class SendLogExceptionStrategy { + + protected final RpcLogSender sender; + + public SendLogExceptionStrategy(RpcLogSender sender){ + this.sender = sender; + } + /** + * Retry count + * @return retry + */ + abstract int retryCount(); + + /** + * + * @param e exception + * @return boolean + */ + abstract RetryDescription onException(Exception e, SendBuffer sendBuffer); + + V doSend(Callable sendOperation, SendBuffer sendBuffer){ + int retryCount = retryCount(); + int count = 0; + RetryDescription retryDescription; + while (++count <= retryCount) { + try { + return sendOperation.call(); + } catch (Exception e) { + retryDescription = onException(e, sendBuffer); + if (Objects.isNull(retryDescription) || !retryDescription.canRetry) { + break; + } + + } + } + return null; + } + + private static class RetryDescription{ + + private final boolean canRetry; + + public RetryDescription(boolean canRetry){ + this.canRetry = canRetry; + } + + public boolean isCanRetry() { + return canRetry; + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java new file mode 100644 index 000000000..06d70407a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -0,0 +1,6 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + + + +public class StreamisRpcLogSender{ +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java similarity index 89% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java index 2f778b231..83e04ec7d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; /** * Abstract sender buffer; @@ -31,6 +31,17 @@ public AbstractSendBuffer(int capacity){ public AbstractSendBuffer(){ this(Integer.MAX_VALUE); } + + @Override + public boolean isReadMode() { + return accessFlag == Flag.READ_MODE; + } + + @Override + public boolean isWriteMode() { + return accessFlag == Flag.WRITE_MODE; + } + @Override public int capacity() { return this.capacity; @@ -49,6 +60,11 @@ public void flip() { this.accessFlag = Flag.READ_MODE; } + @Override + public void rewind() { + position = 0; + } + @Override public void clear() { limit(this.capacity); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java similarity index 99% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index 3587079fa..eefe50ae7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; import java.util.Arrays; import java.util.function.Function; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java similarity index 83% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java index e8bee928b..28e3a6356 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; import java.util.function.Function; @@ -14,6 +14,17 @@ public interface SendBuffer { */ int capacity(); + /** + * Is read mode + * @return boolean + */ + boolean isReadMode(); + + /** + * Is write mode + * @return boolean + */ + boolean isWriteMode(); /** * Scale-up or scale-in * @param newCapacity new capacity @@ -27,10 +38,14 @@ public interface SendBuffer { int remaining(); /** - * transient between write-mode and read-mode + * Transient between write-mode and read-mode */ void flip(); + /** + * Restart from the beginning of window + */ + void rewind(); /** * Clear to reuse the buffer */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java new file mode 100644 index 000000000..b8bba66b8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +public abstract class AbstractHttpLogSender extends AbstractRpcLogSender { + + /** + * Retry strategy + */ + private SendLogExceptionStrategy sendRetryStrategy; + + public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { + super(rpcSenderConfig, cacheSize, sendBufSize); + + } + + @Override + protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) { + + } +} From b1cf64291e1e08e2cf4634bd56adea88bb6fe028 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 11:00:13 +0800 Subject: [PATCH 008/219] fix the bug for adding task for existed jobs. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index bba1dc579..e174947a8 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -281,11 +281,11 @@ public Message addTask(HttpServletRequest req, } else if(streamTask == null) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 - PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, 0, null); - List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark") && job.getStatus() > 0) + PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, null, null); + List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark.")) .collect(Collectors.toList()); if(copyJobs.isEmpty()) { - return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); } int index = 0; while(streamTask == null && index < copyJobs.size()) { @@ -301,7 +301,7 @@ public Message addTask(HttpServletRequest req, } } if(streamTask == null) { - return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); } } streamTask.setStartTime(new Date()); From 247d93bc86d8ba2fd34a75bff4144bd8efead9b1 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 17:31:30 +0800 Subject: [PATCH 009/219] 1. Add the ability for updating task for existed jobs. 2. Spark Streaming Job will not ask Linkis for status. --- .../manager/service/TaskMonitorService.scala | 104 +++++++++--------- .../jobmanager/restful/api/JobRestfulApi.java | 95 +++++++++++++--- 2 files changed, 134 insertions(+), 65 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala index edf36bb94..f6f250e6a 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala @@ -18,6 +18,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import java.util import java.util.Date import java.util.concurrent.{Future, TimeUnit} + import com.google.common.collect.Sets import com.webank.wedatasphere.streamis.jobmanager.launcher.JobLauncherAutoConfiguration import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants @@ -30,7 +31,6 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamTaskMapper} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamTask} import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils - import javax.annotation.{PostConstruct, PreDestroy, Resource} import org.apache.commons.lang.exception.ExceptionUtils import org.apache.linkis.common.exception.ErrorException @@ -84,58 +84,64 @@ class TaskMonitorService extends Logging { return } streamTasks.filter(shouldMonitor).foreach { streamTask => - streamTask.setLastUpdateTime(new Date) - streamTaskMapper.updateTask(streamTask) val job = streamJobMapper.getJobById(streamTask.getJobId) - info(s"Try to update status of StreamJob-${job.getName}.") - val retryHandler = new RetryHandler {} - retryHandler.setRetryNum(3) - retryHandler.setRetryMaxPeriod(2000) - retryHandler.addRetryException(classOf[ErrorException]) - var jobInfo:JobInfo = null - Utils.tryCatch { - jobInfo = retryHandler.retry(refresh(streamTask, jobLaunchManager), s"Task-Monitor-${job.getName}") - } { ex => { - error(s"Fetch StreamJob-${job.getName} failed, maybe the Linkis cluster is wrong, please be noticed!", ex) - val errorMsg = ExceptionUtils.getRootCauseMessage(ex) - if (errorMsg != null && errorMsg.contains("Not exists EngineConn")) { - streamTask.setStatus(JobConf.FLINK_JOB_STATUS_FAILED.getValue) - streamTask.setErrDesc("Not exists EngineConn.") - } else { - // 连续三次还是出现异常，说明Linkis的Manager已经不能正常提供服务，告警并不再尝试获取状态，等待下次尝试 - val users = getAlertUsers(job) - users.add(job.getCreateBy) - alert(jobService.getAlertLevel(job), s"请求LinkisManager失败，Linkis集群出现异常，请关注！影响任务[${job.getName}]", users, streamTask) - } - } - } - streamTaskMapper.updateTask(streamTask) - if(streamTask.getStatus == JobConf.FLINK_JOB_STATUS_FAILED.getValue) { - warn(s"StreamJob-${job.getName} is failed, please be noticed.") - var extraMessage = "" - Option(jobInfo) match { - case Some(flinkJobInfo: FlinkJobInfo) => - extraMessage = s",${flinkJobInfo.getApplicationId}" - case _ => - } - // Need to add restart feature if user sets the restart parameters. - var alertMsg = s"Streamis 流式应用[${job.getName}${extraMessage}]已经失败, 请登陆Streamis查看应用日志." - this.streamJobConfMapper.getRawConfValue(job.getId, JobConfKeyConstants.FAIL_RESTART_SWITCH.getValue) match { - case "ON" => - alertMsg = s"${alertMsg} 现将自动拉起该应用" - Utils.tryCatch{ - info(s"Start to reLaunch the StreamisJob [${job.getName}], now to submit and schedule it...") - // Use submit user to start job - val future: Future[String] = streamTaskService.asyncExecute(job.getId, 0L, job.getSubmitUser, true) - }{ - case e:Exception => - warn(s"Fail to reLaunch the StreamisJob [${job.getName}]", e) - } - case _ => - } + if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES.getValue.contains(job.getJobType)) { val userList = Sets.newHashSet(job.getSubmitUser, job.getCreateBy) userList.addAll(getAlertUsers(job)) + val alertMsg = s"Spark Streaming应用[${job.getName}]已经超过 ${Utils.msDurationToString(System.currentTimeMillis - streamTask.getLastUpdateTime.getTime)} 没有更新状态, 请及时确认应用是否正常！" alert(jobService.getAlertLevel(job), alertMsg, new util.ArrayList[String](userList), streamTask) + } else { + streamTask.setLastUpdateTime(new Date) + streamTaskMapper.updateTask(streamTask) + info(s"Try to update status of StreamJob-${job.getName}.") + val retryHandler = new RetryHandler {} + retryHandler.setRetryNum(3) + retryHandler.setRetryMaxPeriod(2000) + retryHandler.addRetryException(classOf[ErrorException]) + var jobInfo:JobInfo = null + Utils.tryCatch { + jobInfo = retryHandler.retry(refresh(streamTask, jobLaunchManager), s"Task-Monitor-${job.getName}") + } { ex => + error(s"Fetch StreamJob-${job.getName} failed, maybe the Linkis cluster is wrong, please be noticed!", ex) + val errorMsg = ExceptionUtils.getRootCauseMessage(ex) + if (errorMsg != null && errorMsg.contains("Not exists EngineConn")) { + streamTask.setStatus(JobConf.FLINK_JOB_STATUS_FAILED.getValue) + streamTask.setErrDesc("Not exists EngineConn.") + } else { + // 连续三次还是出现异常，说明Linkis的Manager已经不能正常提供服务，告警并不再尝试获取状态，等待下次尝试 + val users = getAlertUsers(job) + users.add(job.getCreateBy) + alert(jobService.getAlertLevel(job), s"请求LinkisManager失败，Linkis集群出现异常，请关注！影响任务[${job.getName}]", users, streamTask) + } + } + streamTaskMapper.updateTask(streamTask) + if(streamTask.getStatus == JobConf.FLINK_JOB_STATUS_FAILED.getValue) { + warn(s"StreamJob-${job.getName} is failed, please be noticed.") + var extraMessage = "" + Option(jobInfo) match { + case Some(flinkJobInfo: FlinkJobInfo) => + extraMessage = s",${flinkJobInfo.getApplicationId}" + case _ => + } + // Need to add restart feature if user sets the restart parameters. + var alertMsg = s"Streamis 流式应用[${job.getName}${extraMessage}]已经失败, 请登陆Streamis查看应用日志." + this.streamJobConfMapper.getRawConfValue(job.getId, JobConfKeyConstants.FAIL_RESTART_SWITCH.getValue) match { + case "ON" => + alertMsg = s"${alertMsg} 现将自动拉起该应用" + Utils.tryCatch{ + info(s"Start to reLaunch the StreamisJob [${job.getName}], now to submit and schedule it...") + // Use submit user to start job + val future: Future[String] = streamTaskService.asyncExecute(job.getId, 0L, job.getSubmitUser, true) + }{ + case e:Exception => + warn(s"Fail to reLaunch the StreamisJob [${job.getName}]", e) + } + case _ => + } + val userList = Sets.newHashSet(job.getSubmitUser, job.getCreateBy) + userList.addAll(getAlertUsers(job)) + alert(jobService.getAlertLevel(job), alertMsg, new util.ArrayList[String](userList), streamTask) + } } } info("All StreamTasks status have updated.") diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index e174947a8..37ed9bf1a 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -22,6 +22,7 @@ import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.JobInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobStateInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo; import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf; @@ -51,6 +52,7 @@ import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; @RequestMapping(path = "/streamis/streamJobManager/job") @@ -277,6 +279,7 @@ public Message addTask(HttpServletRequest req, if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); streamTaskService.updateTask(streamTask); } else if(streamTask == null) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo @@ -306,21 +309,65 @@ public Message addTask(HttpServletRequest req, } streamTask.setStartTime(new Date()); streamTask.setLastUpdateTime(new Date()); + StreamTask finalStreamTask = streamTask; + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + flinkJobInfo.setApplicationId(appId); + flinkJobInfo.setApplicationUrl(appUrl); + flinkJobInfo.setName(jobName); + flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); + StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); + streamTaskService.updateTask(finalStreamTask); + LOG.info("Streamis Job {} has added a new task successfully.", jobName); + return Message.ok(); + }); + } + + private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Function flinkJobInfoFunction) { FlinkJobInfo flinkJobInfo; try { - flinkJobInfo = DWSHttpClient.jacksonJson().readValue(streamTask.getLinkisJobInfo(), FlinkJobInfo.class); + flinkJobInfo = DWSHttpClient.jacksonJson().readValue(flinkJobInfoStr, FlinkJobInfo.class); } catch (JsonProcessingException e) { - LOG.error("Job {} deserialize the jobInfo from history Job failed!", jobName, e); - return Message.error("Deserialize the jobInfo from history Job failed!"); - } - flinkJobInfo.setApplicationId(appId); - flinkJobInfo.setApplicationUrl(appUrl); - flinkJobInfo.setName(jobName); - flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); - StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); - streamTaskService.updateTask(streamTask); - LOG.info("Streamis Job {} has added a new task successfully.", jobName); - return Message.ok(); + LOG.error("Job {} deserialize the flinkJobInfo string to object failed!", jobName, e); + return Message.error("Deserialize the flinkJobInfo string to object failed!"); + } + return flinkJobInfoFunction.apply(flinkJobInfo); + } + + @RequestMapping(path = "/updateTask", method = RequestMethod.GET) + public Message updateTask(HttpServletRequest req, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "metrics") String metrics) { + String username = SecurityFilter.getLoginUsername(req); + LOG.info("User {} try to update task for Streamis job {} with appId: {}.", username, jobName, appId); + List streamJobs = streamJobService.getJobByName(jobName); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to update task."); + } + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + if(streamTask == null) { + LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); + return Message.ok("not exists running task, ignore it."); + } + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if(!flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to update its metrics.", + jobName, flinkJobInfo.getApplicationId(), appId); + return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); + } + JobStateInfo jobStateInfo = new JobStateInfo(); + jobStateInfo.setTimestamp(System.currentTimeMillis()); + jobStateInfo.setLocation(metrics); + flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); + StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); + streamTaskService.updateTask(streamTask); + LOG.info("Streamis Job {} has updated the task metrics successfully.", jobName); + return Message.ok(); + }); } @RequestMapping(path = "/stopTask", method = RequestMethod.GET) @@ -345,13 +392,23 @@ public Message stopTask(HttpServletRequest req, // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTaskService.updateTask(streamTask); + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if(flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App itself."); + streamTaskService.updateTask(streamTask); + return Message.ok(); + } else { + LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to stop it.", + jobName, flinkJobInfo.getApplicationId(), appId); + return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); + } + }); } else { LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", jobName); + return Message.ok(); } - return Message.ok(); } @RequestMapping(path = "/progress", method = RequestMethod.GET) @@ -411,6 +468,12 @@ public Message getLog(HttpServletRequest req, logType = StringUtils.isBlank(logType) ? "client" : logType; String username = SecurityFilter.getLoginUsername(req); StreamJob streamJob = this.streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType()) && + "client".equals(logType)) { + return Message.error("Job " + streamJob.getName() + " is not supported to get client logs."); + } if (!streamJobService.hasPermission(streamJob, username) && !this.privilegeService.hasAccessPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to fetch logs from StreamJob [" + jobId + "]"); From f0016122d02d3e3af560225d6336217bccfffdeb Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 20 Oct 2022 19:12:02 +0800 Subject: [PATCH 010/219] RPC(Http) module classes in collector --- .../streamis-job-log-collector/pom.xml | 20 +++ .../sender/AbstractRpcLogSender.java | 77 +++++--- .../log/collector/sender/RpcSenderConfig.java | 170 +++++++++++++++++- .../sender/SendLogCacheConsumer.java | 19 +- .../sender/SendLogExceptionStrategy.java | 6 +- .../sender/http/AbstractHttpLogSender.java | 34 +++- .../collector/sender/http/HttpClientTool.java | 71 ++++++++ .../log/collector/sender/http/Json.java | 130 ++++++++++++++ .../sender/http/entities/Resource.java | 17 ++ .../http/request/AbstractHttpAction.java | 62 +++++++ .../sender/http/request/EntityPostAction.java | 80 +++++++++ .../sender/http/request/GetAction.java | 4 + .../sender/http/request/HttpAction.java | 38 ++++ .../sender/http/request/PostAction.java | 41 +++++ .../jobmanager/log/entities/LogElement.java | 3 +- .../log/entities/StreamisLogEvents.java | 19 ++ 16 files changed, 752 insertions(+), 39 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index 9549d3439..bbc6a69c0 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -14,6 +14,9 @@ 8 8 + 4.5.13 + 4.5.4 + 2.13.2.2 @@ -22,5 +25,22 @@ streamis-job-log-common 0.2.0 + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpmine.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index 3c010878b..6b4d36580 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -5,10 +5,7 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; @@ -139,8 +136,17 @@ private class RpcLogContext{ */ private int consumers = 0; + /** + * Futures of consumers + */ + private final Map> sendLogCacheConsumers = new ConcurrentHashMap<>(); + /** + * Context lock + */ + private final ReentrantLock ctxLock; public RpcLogContext(SendLogCache logCache){ this.logCache = logCache; + this.ctxLock = new ReentrantLock(); this.consumePool = new ThreadPoolExecutor(0, maxCacheConsume, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), new ThreadFactory() { @@ -161,33 +167,58 @@ public Thread newThread(Runnable r) { }); } - public synchronized void startCacheConsumer(){ - if (consumers >= maxCacheConsume){ - throw new IllegalStateException("Over the limit number of cache consumers: [" + maxCacheConsume + "]"); - } - String id = UUID.randomUUID().toString(); - SendBuffer sendBuffer = new ImmutableSendBuffer<>(sendBufSize); - this.consumePool.submit(new SendLogCacheConsumer(id, logCache, sendBuffer, rpcSenderConfig) { - @Override - protected void onFlushAndSend(SendBuffer sendBuffer) { - // First to aggregate the buffer - E aggEntity = aggregateBuffer(sendBuffer); - Optional.ofNullable(getSendLogExceptionStrategy()).ifPresent( - strategy -> strategy.doSend(() -> { - doSend(aggEntity, rpcSenderConfig); - return null; - }, sendBuffer)); + public void startCacheConsumer(){ + this.ctxLock.lock(); + try { + if (consumers >= maxCacheConsume) { + throw new IllegalStateException("Over the limit number of cache consumers: [" + maxCacheConsume + "]"); } - }); - this.consumers ++; + String id = UUID.randomUUID().toString(); + SendBuffer sendBuffer = new ImmutableSendBuffer<>(sendBufSize); + SendLogCacheConsumer consumer = new SendLogCacheConsumer(id, logCache, sendBuffer, rpcSenderConfig) { + @Override + protected void onFlushAndSend(SendBuffer sendBuffer) { + // First to aggregate the buffer + E aggEntity = aggregateBuffer(sendBuffer); + Optional.ofNullable(getSendLogExceptionStrategy()).ifPresent( + strategy -> strategy.doSend(() -> { + doSend(aggEntity, rpcSenderConfig); + return null; + }, sendBuffer)); + } + }; + Future future = this.consumePool.submit(consumer); + consumer.setFuture(future); + sendLogCacheConsumers.put(id, consumer); + this.consumers++; + } finally { + this.ctxLock.unlock(); + } } public SendLogCache getLogCache(){ return this.logCache; } - public void destroyCacheConsumer(){ + /** + * Destroy cache consumer + * @param id id + */ + public void destroyCacheConsumer(String id){ + SendLogCacheConsumer consumer = sendLogCacheConsumers.remove(id); + consumer.shutdown(); + } + /** + * Destroy all the consumers + */ + public void destroyCacheConsumers(){ + this.ctxLock.lock(); + try { + sendLogCacheConsumers.forEach( (key, consumer)-> consumer.shutdown()); + } finally { + this.ctxLock.unlock(); + } } } /** diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java index a0d3a77e5..d7a307f2a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java @@ -8,27 +8,181 @@ public class RpcSenderConfig { /** * Send address */ - private String sendAddress; + private String address; + + /** + * Key of token-code + */ + private String tokenCodeKey = "Token-Code"; + + /** + * Key of token-user + */ + private String tokenUserKey = "Token-User"; + + /** + * Token user + */ + private String tokenUser = System.getProperty("user.name"); + + /** + * Token code + */ + private String tokenCode = "STREAM_LOG"; + + /** + * Timeout of connecting + */ + private int connectionTimeout = 3000; + + /** + * Timeout of reading from socket + */ + private int socketTimeout = 15000; + /** * Size of send cache */ - private int sendCacheSize = 150; + private int cacheSize = 150; /** * Size of send buffer */ - private int sendBufferSize = 50; + private int bufferSize = 50; /** * Expire time of send buffer */ - private long sendBufferExpireTimeInSec = -1; + private long bufferExpireTimeInSec = -1; + + /** + * Retry count of sending + */ + private int sendRetryCnt = 3; + + /** + * The time for server recovery + */ + private int serverRecoveryTimeInSec = 5; + /** + * Retry max delay time of sender + */ + private int maxDelayTimeInSec; + + /** + * Max number of consuming thread + */ + private int maxConsumeThread = 10; + + public long getBufferExpireTimeInSec() { + return bufferExpireTimeInSec; + } + + public void setBufferExpireTimeInSec(long bufferExpireTimeInSec) { + this.bufferExpireTimeInSec = bufferExpireTimeInSec; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public int getCacheSize() { + return cacheSize; + } + + public void setCacheSize(int cacheSize) { + this.cacheSize = cacheSize; + } + + public int getBufferSize() { + return bufferSize; + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + public int getSendRetryCnt() { + return sendRetryCnt; + } + + public void setSendRetryCnt(int sendRetryCnt) { + this.sendRetryCnt = sendRetryCnt; + } + + public int getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public int getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public int getMaxConsumeThread() { + return maxConsumeThread; + } + + public void setMaxConsumeThread(int maxConsumeThread) { + this.maxConsumeThread = maxConsumeThread; + } + + public String getTokenCodeKey() { + return tokenCodeKey; + } + + public void setTokenCodeKey(String tokenCodeKey) { + this.tokenCodeKey = tokenCodeKey; + } + + public String getTokenCode() { + return tokenCode; + } + + public void setTokenCode(String tokenCode) { + this.tokenCode = tokenCode; + } + + public int getMaxDelayTimeInSec() { + return maxDelayTimeInSec; + } + + public void setMaxDelayTimeInSec(int maxDelayTimeInSec) { + this.maxDelayTimeInSec = maxDelayTimeInSec; + } + + public String getTokenUserKey() { + return tokenUserKey; + } + + public void setTokenUserKey(String tokenUserKey) { + this.tokenUserKey = tokenUserKey; + } + + public String getTokenUser() { + return tokenUser; + } + + public void setTokenUser(String tokenUser) { + this.tokenUser = tokenUser; + } - public long getSendBufferExpireTimeInSec() { - return sendBufferExpireTimeInSec; + public int getServerRecoveryTimeInSec() { + return serverRecoveryTimeInSec; } - public void setSendBufferExpireTimeInSec(long sendBufferExpireTimeInSec) { - this.sendBufferExpireTimeInSec = sendBufferExpireTimeInSec; + public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { + this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index f0d8f5866..e851943a7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -3,6 +3,7 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; /** @@ -29,14 +30,20 @@ public abstract class SendLogCacheConsumer implements Runn private final SendBuffer sendBuffer; private final String id; + + /** + * Future for execution + */ + private Future future; + public SendLogCacheConsumer(String id, SendLogCache cache, SendBuffer sendBuffer, RpcSenderConfig rpcSenderConfig){ this.id = id; this.cache = cache; this.sendBuffer = sendBuffer; - this.bufferExpireTimeInMills = rpcSenderConfig.getSendBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS - .toMillis(rpcSenderConfig.getSendBufferExpireTimeInSec()) : -1; + this.bufferExpireTimeInMills = rpcSenderConfig.getBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS + .toMillis(rpcSenderConfig.getBufferExpireTimeInSec()) : -1; } @@ -88,6 +95,14 @@ public void shutdown(){ this.isTerminated = true; } + public Future getFuture() { + return future; + } + + public void setFuture(Future future) { + this.future = future; + } + private long requireNewFlushTime(){ return bufferExpireTimeInMills > 0 ? System.currentTimeMillis() + bufferExpireTimeInMills : -1; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java index d3c2fafd9..c8952ff1d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java @@ -20,14 +20,14 @@ public SendLogExceptionStrategy(RpcLogSender sender){ * Retry count * @return retry */ - abstract int retryCount(); + public abstract int retryCount(); /** * * @param e exception * @return boolean */ - abstract RetryDescription onException(Exception e, SendBuffer sendBuffer); + public abstract RetryDescription onException(Exception e, SendBuffer sendBuffer); V doSend(Callable sendOperation, SendBuffer sendBuffer){ int retryCount = retryCount(); @@ -47,7 +47,7 @@ V doSend(Callable sendOperation, SendBuffer sendBuffer){ return null; } - private static class RetryDescription{ + protected static class RetryDescription{ private final boolean canRetry; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index b8bba66b8..7f2c68d04 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -3,22 +3,54 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.http.client.HttpClient; + +import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractHttpLogSender extends AbstractRpcLogSender { /** * Retry strategy */ - private SendLogExceptionStrategy sendRetryStrategy; + private SendLogExceptionStrategy sendRetryStrategy; + + /** + * Exception counter + */ + private AtomicInteger exceptionCounter = new AtomicInteger(); + /** + * Hold the global http client + */ + private HttpClient globalHttpClient; public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { super(rpcSenderConfig, cacheSize, sendBufSize); + this.globalHttpClient = HttpClientTool.createHttpClient(rpcSenderConfig); + this.sendRetryStrategy = new SendLogExceptionStrategy(this) { + @Override + public int retryCount() { + return rpcSenderConfig.getSendRetryCnt(); + } + + @Override + public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { + return null; + } + }; } @Override protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) { + if (aggregatedEntity instanceof LogElement){ + long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); + if (System.currentTimeMillis() - timestamp > rpcSenderConfig.getMaxDelayTimeInSec() * 1000L){ + // Abort the entity + return; + } + } } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java new file mode 100644 index 000000000..df4eb31ea --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import org.apache.http.Header; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Http clients + */ +public class HttpClientTool { + + /** + * Connect timeout + */ + public static final int DEFAULT_CONNECT_TIMEOUT = 3000; + + /** + * Socket timeout + */ + public static final int DEFAULT_SOCKET_TIMEOUT = 15000; + + /** + * Max connections + */ + public static final int DEFAULT_MAX_CONN = 10; + + /** + * Create http client + * @param rpcSenderConfig rpc sender config + * @return http client + */ + public static HttpClient createHttpClient(RpcSenderConfig rpcSenderConfig){ + int connectTimeout = rpcSenderConfig.getConnectionTimeout() > 0? rpcSenderConfig.getConnectionTimeout() : DEFAULT_CONNECT_TIMEOUT; + int socketTimeout = rpcSenderConfig.getSocketTimeout() > 0? rpcSenderConfig.getSocketTimeout() : DEFAULT_SOCKET_TIMEOUT; + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(connectTimeout) + .setConnectionRequestTimeout(socketTimeout) + .setSocketTimeout(socketTimeout) + .build(); + int maxConn = rpcSenderConfig.getMaxConsumeThread() > 0? rpcSenderConfig.getMaxConsumeThread() : DEFAULT_MAX_CONN; + HttpClientBuilder clientBuilder = HttpClients.custom(); + String tokenValue = rpcSenderConfig.getTokenCode(); + List
defaultHeaders = new ArrayList<>(); + if (null != tokenValue && !tokenValue.trim().equals("")){ + defaultHeaders.add(new BasicHeader(rpcSenderConfig.getTokenCodeKey(), tokenValue)); + } + clientBuilder.setDefaultRequestConfig(requestConfig).setDefaultHeaders(defaultHeaders) + .useSystemProperties().setMaxConnTotal(maxConn); + CloseableHttpClient httpClient = clientBuilder.build(); + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + try { + httpClient.close(); + } catch (IOException e) { + // Ignore + } + } + })); + return httpClient; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java new file mode 100644 index 000000000..8c493d042 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java @@ -0,0 +1,130 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.*; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; + +/** + * Json utils + */ +public class Json { + private static final String PREFIX = "["; + private static final String SUFFIX = "]"; + + private static ObjectMapper mapper; + + static{ + mapper = new ObjectMapper(); + mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); + mapper.configure(DeserializationFeature.READ_ENUMS_USING_TO_STRING, true); + mapper.configure(SerializationFeature.WRITE_ENUMS_USING_TO_STRING, true); + mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); + //empty beans allowed + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + //cancel to scape non ascii + mapper.configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, false); + } + private Json(){} + + @SuppressWarnings("unchecked") + public static T fromJson(String json, Class clazz, Class... parameters) throws Exception{ + if(null != json && !json.trim().equals("")){ + try{ + if(parameters.length > 0){ + return (T)mapper.readValue(json, mapper.getTypeFactory().constructParametricType(clazz, parameters)); + } + if(json.startsWith(PREFIX) + && json.endsWith(SUFFIX)){ + JavaType javaType = mapper.getTypeFactory() + .constructParametricType(ArrayList.class, clazz); + return mapper.readValue(json, javaType); + } + return (T)mapper.readValue(json, clazz); + } catch (Exception e) { + String message = "Unable to deserialize to object from string(json) in type: [" + + (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; + throw new RuntimeException(message, e); + } + } + return null; + } + + public static T fromJson(InputStream stream, Class clazz, Class... parameters) throws Exception{ + StringBuilder builder = new StringBuilder(); + String jsonStr = null; + try{ + BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); + while((jsonStr = reader.readLine()) != null){ + builder.append(jsonStr); + } + reader.close(); + }catch(Exception e){ + String message = "Unable to deserialize to object from stream(json) in type: [" + + (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; + throw new RuntimeException(message, e); + } + return fromJson(builder.toString(), clazz, parameters); + } + + public static String toJson(Object obj, Class model) { + ObjectWriter writer = mapper.writer(); + if(null != obj){ + try{ + if(null != model){ + writer = writer.withView(model); + } + return writer.writeValueAsString(obj); + } catch (JsonProcessingException e) { + String message = "Unable to serialize the object in type: ["+ (null != model ? model.getSimpleName() : "UNKNOWN") + "]"; + throw new RuntimeException(message, e); + } + } + return null; + } + + /** + * Convert object using serialization and deserialization + * + * @param simpleObj simpleObj + * @param tClass type class + * @param T + * @return result + */ + @SuppressWarnings("unchecked") + public static T convert(Object simpleObj, Class tClass, Class... parameters) { + try { + if (parameters.length > 0) { + return mapper.convertValue(simpleObj, mapper.getTypeFactory().constructParametricType(tClass, parameters)); + } + return (T) mapper.convertValue(simpleObj, tClass); + } catch (Exception e) { + String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + + ", " + tClass.getSimpleName() + ": "+ Class.class + ", ...: " + Class.class + ")"; + throw new RuntimeException(message, e); + } + } + + public static T convert(Object simpleObj, JavaType javaType) { + try { + return mapper.convertValue(simpleObj, javaType); + } catch (Exception e) { + String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + + ", " + javaType.getTypeName() + ": "+ JavaType.class + ")"; + throw new RuntimeException(message, e); + } + } + + public static ObjectMapper getMapper(){ + return mapper; + } +} + diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java new file mode 100644 index 000000000..eaa355e92 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.entities; + +import java.io.File; +import java.util.List; + +/** + * Entity with resources + */ +public interface Resource { + + /** + * Resources related + * @return file list + */ + List getResources(); + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java new file mode 100644 index 000000000..8b0472e08 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpRequestBase; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + +/** + * Abstract implement + * @param + */ +public abstract class AbstractHttpAction implements HttpAction { + + protected String uri; + + protected String user; + + public AbstractHttpAction(String uri){ + this.uri = uri; + } + + @Override + public String uri() { + return uri; + } + + /** + * Request method + * @return method + */ + protected abstract T getRequestMethod(); + + private Map requestHeaders = new HashMap<>(); + + private Map requestPayload = new HashMap<>(); + + @Override + public Map getRequestHeaders() { + return this.requestHeaders; + } + + @Override + public Map getRequestPayload() { + return this.requestPayload; + } + + @Override + public HttpResponse execute(HttpClient httpClient) throws IOException { + HttpRequestBase requestBase = getRequestMethod(); + try{ + requestBase.setURI(new URI(uri)); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("URI maybe has wrong format", e); + } + return httpClient.execute(requestBase); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java new file mode 100644 index 000000000..f8bfc5e4c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java @@ -0,0 +1,80 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.entities.Resource; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Post action for entity + */ +public class EntityPostAction extends PostAction{ + + private static final String RESOURCE_FIELD_NAME = "resources"; + + private T entity; + + public EntityPostAction(String uri, T entity) { + super(uri); + if (Objects.isNull(entity)){ + throw new IllegalArgumentException("Entity cannot be null in action"); + } + this.entity = entity; + } + + @Override + HttpEntity getHttpEntity() { + try { + if (entity instanceof Resource) { + MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); + String entityStr = Json.toJson(entity, null); + JsonNode rootNode = Json.getMapper().readTree(entityStr); + addEntityBody(entityBuilder, "", rootNode); + Optional.ofNullable(((Resource) entity).getResources()).ifPresent(resources -> { + for(int i = 0; i < resources.size(); i ++){ + entityBuilder.addBinaryBody(RESOURCE_FIELD_NAME + "[" + i +"]", resources.get(i)); + } + }); + return entityBuilder.build(); + } else { + StringEntity stringEntity; + stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(entity, null)), "UTF-8"); + stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); + return stringEntity; + } + } catch (Exception e) { + throw new IllegalArgumentException("Cannot deserialize the entity: " + entity.getClass().getName(), e); + } + } + + private void addEntityBody(MultipartEntityBuilder builder, String prefix, JsonNode node){ + if (node instanceof ObjectNode){ + if (null != prefix && !prefix.trim().equals("")){ + prefix += "."; + } + Iterator> fields = node.fields(); + while (fields.hasNext()){ + Map.Entry entry = fields.next(); + addEntityBody(builder, prefix + entry.getKey(), entry.getValue()); + } + } else if (node instanceof ArrayNode){ + ArrayNode arrayNode = ((ArrayNode)node); + for (int i = 0 ; i < arrayNode.size(); i ++){ + addEntityBody(builder, prefix + "[" + i + "]", arrayNode.get(i)); + } + } else if (node instanceof ValueNode){ + ContentType strContent = ContentType.create("text/plain", StandardCharsets.UTF_8); + System.out.println("p: " + prefix + ", data: " + node.asText()); + builder.addTextBody(prefix, node.asText(), strContent); + } + } + +} \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java new file mode 100644 index 000000000..f5a8a5fef --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +public class GetAction { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java new file mode 100644 index 000000000..87435f8a3 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; + +import java.io.IOException; +import java.util.Map; + +/** + * Http action + */ +public interface HttpAction { + + /** + * URI path + * @return path + */ + String uri(); + + /** + * Request headers + * @return map + */ + Map getRequestHeaders(); + + /** + * Request pay load(body) + * @return map + */ + Map getRequestPayload(); + + /** + * Execute http action + * @return http response + */ + HttpResponse execute(HttpClient httpClient) throws IOException; + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java new file mode 100644 index 000000000..9b3ea003f --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java @@ -0,0 +1,41 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; + +import java.util.Objects; + +/** + * Post action + */ +public class PostAction extends AbstractHttpAction { + + public PostAction(String uri) { + super(uri); + } + + /** + * Http entity (use stringEntity and "application/json" default) + * @return http entity + */ + HttpEntity getHttpEntity(){ + StringEntity stringEntity; + try{ + stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(getRequestPayload(), null)), "UTF-8"); + } catch (Exception e){ + throw new IllegalArgumentException("Cannot deserialize the requestPayLoad: " + getRequestPayload().toString(), e); + } + stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); + return stringEntity; + } + + @Override + protected HttpPost getRequestMethod() { + HttpPost httpPost = new HttpPost(); + httpPost.setEntity(getHttpEntity()); + return httpPost; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index a2b0a7807..8a497b61e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -15,10 +15,9 @@ public interface LogElement { /** * Log time - * @param unit unit * @return log time */ - long getLogTime(TimeUnit unit); + long getLogTimeStamp(); /** * Get content diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java new file mode 100644 index 000000000..f194fb289 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.entities; + +public class StreamisLogEvents implements LogElement{ + @Override + public int getSequenceId() { + return 0; + } + + @Override + public long getLogTimeStamp() { + return 0; + } + + + @Override + public String[] getContents() { + return new String[0]; + } +} From 06a4f63d95ad00d7428fb7ce9527729e87f7fb7d Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 21:15:11 +0800 Subject: [PATCH 011/219] 1. Add the ability for updating task for existed jobs. 2. Spark Streaming Job will not ask Linkis for status. --- .../jobmanager/restful/api/JobRestfulApi.java | 139 +++++++++++++----- 1 file changed, 104 insertions(+), 35 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 37ed9bf1a..7b9f5d9c2 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -39,6 +39,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.utils.Utils; import org.apache.linkis.httpclient.dws.DWSHttpClient; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; @@ -50,7 +51,6 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import java.io.IOException; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -194,44 +194,107 @@ public Message killJob(HttpServletRequest req, @RequestMapping(path = "/details", method = RequestMethod.GET) public Message detailsJob(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, - @RequestParam(value = "version", required = false) String version) throws JobException { + @RequestParam(value = "version", required = false) String version) throws JobException, JsonProcessingException { if (jobId == null) { JobExceptionManager.createException(30301, "jobId"); } - // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. + + StreamJob streamJob = streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } + FlinkJobInfo flinkJobInfo = streamTaskService.getTaskJobInfo(jobId,version); JobDetailsVo jobDetailsVO = new JobDetailsVo(); List dataNumberDTOS = new ArrayList<>(); - JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); - dataNumberDTO.setDataName("kafka topic"); - dataNumberDTO.setDataNumber(109345); - dataNumberDTOS.add(dataNumberDTO); - List loadConditionDTOs = new ArrayList<>(); - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType("jobManager"); - loadConditionDTO.setHost("localhost"); - loadConditionDTO.setMemory("1.5"); - loadConditionDTO.setTotalMemory("2.0"); - loadConditionDTO.setGcLastTime("2020-08-01"); - loadConditionDTO.setGcLastConsume("1"); - loadConditionDTO.setGcTotalTime("2min"); - loadConditionDTOs.add(loadConditionDTO); - List realTimeTrafficDTOS = new ArrayList<>(); - JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); - realTimeTrafficDTO.setSourceKey("kafka topic"); - realTimeTrafficDTO.setSourceSpeed("100 Records/S"); - realTimeTrafficDTO.setTransformKey("transform"); - realTimeTrafficDTO.setSinkKey("hbase key"); - realTimeTrafficDTO.setSinkSpeed("10 Records/S"); - realTimeTrafficDTOS.add(realTimeTrafficDTO); - + JobStateInfo[] jobStateInfos = flinkJobInfo.getJobStates(); + if(JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType()) || + (jobStateInfos == null || jobStateInfos.length == 0)) { + // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. + JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); + dataNumberDTO.setDataName("kafka topic"); + dataNumberDTO.setDataNumber(109345); + dataNumberDTOS.add(dataNumberDTO); + + JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); + loadConditionDTO.setType("jobManager"); + loadConditionDTO.setHost("localhost"); + loadConditionDTO.setMemory("1.5"); + loadConditionDTO.setTotalMemory("2.0"); + loadConditionDTO.setGcLastTime("2020-08-01"); + loadConditionDTO.setGcLastConsume("1"); + loadConditionDTO.setGcTotalTime("2min"); + loadConditionDTOs.add(loadConditionDTO); + + JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); + realTimeTrafficDTO.setSourceKey("kafka topic"); + realTimeTrafficDTO.setSourceSpeed("100 Records/S"); + realTimeTrafficDTO.setTransformKey("transform"); + realTimeTrafficDTO.setSinkKey("hbase key"); + realTimeTrafficDTO.setSinkSpeed("10 Records/S"); + realTimeTrafficDTOS.add(realTimeTrafficDTO); - jobDetailsVO.setLinkisJobInfo(streamTaskService.getTaskJobInfo(jobId,version)); + } else { + String metricsStr = jobStateInfos[0].getLocation(); + Map metricsMap = DWSHttpClient.jacksonJson().readValue(metricsStr, Map.class); + JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); + dataNumberDTO.setDataName("waitingBatchs"); + dataNumberDTO.setDataNumber(Math.toIntExact((Long) metricsMap.get("waitingBatchs"))); + dataNumberDTOS.add(dataNumberDTO); + List> executors = (List>) metricsMap.get("executors"); + if(executors != null && !executors.isEmpty()) { + executors.forEach(map -> { + JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); + loadConditionDTO.setType((String) map.get("type")); + loadConditionDTO.setHost((String) map.get("host")); + loadConditionDTO.setMemory((String) map.get("memory")); + loadConditionDTO.setTotalMemory((String) map.get("totalMemory")); + loadConditionDTO.setGcLastTime((String) map.get("gcLastTime")); + loadConditionDTO.setGcLastConsume((String) map.get("gcLastConsume")); + loadConditionDTO.setGcTotalTime((String) map.get("gcTotalTime")); + loadConditionDTOs.add(loadConditionDTO); + }); + } else { + JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); + loadConditionDTO.setType("sparkAppMaster"); + loadConditionDTO.setHost(""); + loadConditionDTO.setMemory(""); + loadConditionDTO.setTotalMemory(""); + loadConditionDTO.setGcLastTime(""); + loadConditionDTO.setGcLastConsume(""); + loadConditionDTO.setGcTotalTime(""); + loadConditionDTOs.add(loadConditionDTO); + } + JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); + List> batchMetrics = (List>) metricsMap.get("batchMetrics"); + if(batchMetrics != null && !batchMetrics.isEmpty()) { + batchMetrics.stream().max(Comparator.comparing(map -> String.valueOf(map.get("batchTime")))).ifPresent(batchMetric -> { + realTimeTrafficDTO.setSourceKey((String) metricsMap.getOrDefault("source", "")); + realTimeTrafficDTO.setSourceSpeed(batchMetric.get("inputRecords") + " Records"); + realTimeTrafficDTO.setTransformKey("processing"); + realTimeTrafficDTO.setSinkKey((String) metricsMap.getOrDefault("sink", "")); + String totalDelay = ""; + if(batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) { + totalDelay = Utils.msDurationToString((long) batchMetric.get("totalDelay")) + " totalDelay"; + } else if(batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) { + totalDelay = Utils.msDurationToString((long) batchMetric.get("taskExecuteTime")) + " executeTime(Last Batch)"; + } + realTimeTrafficDTO.setSinkSpeed(totalDelay); + }); + } else { + realTimeTrafficDTO.setSourceKey(""); + realTimeTrafficDTO.setSourceSpeed(" Records/S"); + realTimeTrafficDTO.setTransformKey(""); + realTimeTrafficDTO.setSinkKey(""); + realTimeTrafficDTO.setSinkSpeed(" Records/S"); + } + realTimeTrafficDTOS.add(realTimeTrafficDTO); + } jobDetailsVO.setDataNumber(dataNumberDTOS); jobDetailsVO.setLoadCondition(loadConditionDTOs); jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); - + jobDetailsVO.setLinkisJobInfo(flinkJobInfo); return Message.ok().data("details", jobDetailsVO); } @@ -276,12 +339,7 @@ public Message addTask(HttpServletRequest req, } // 如果存在正在运行的，先将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); - if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTask.setErrDesc("stopped by App's new task."); - streamTaskService.updateTask(streamTask); - } else if(streamTask == null) { + if(streamTask == null) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, null, null); @@ -306,6 +364,17 @@ public Message addTask(HttpServletRequest req, if(streamTask == null) { return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); } + } else { + if(JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); + streamTaskService.updateTask(streamTask); + } + StreamTask newStreamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setId(newStreamTask.getId()); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue()); + streamTask.setSubmitUser(username); } streamTask.setStartTime(new Date()); streamTask.setLastUpdateTime(new Date()); @@ -359,7 +428,7 @@ public Message updateTask(HttpServletRequest req, jobName, flinkJobInfo.getApplicationId(), appId); return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); } - JobStateInfo jobStateInfo = new JobStateInfo(); + JobStateInfo jobStateInfo = new JobStateInfo(); jobStateInfo.setTimestamp(System.currentTimeMillis()); jobStateInfo.setLocation(metrics); flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); From f5dc4ecd44bca73444ea4437e05a4dfcfce9395c Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 21:24:47 +0800 Subject: [PATCH 012/219] 1. Add the ability for updating task for existed jobs. 2. Spark Streaming Job will not ask Linkis for status. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 7b9f5d9c2..57d5473b5 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -373,7 +373,9 @@ public Message addTask(HttpServletRequest req, } StreamTask newStreamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); streamTask.setId(newStreamTask.getId()); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue()); + streamTask.setVersion(newStreamTask.getVersion()); + streamTask.setErrDesc(""); + streamTask.setStatus(newStreamTask.getStatus()); streamTask.setSubmitUser(username); } streamTask.setStartTime(new Date()); @@ -383,7 +385,7 @@ public Message addTask(HttpServletRequest req, flinkJobInfo.setApplicationId(appId); flinkJobInfo.setApplicationUrl(appUrl); flinkJobInfo.setName(jobName); - flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); + flinkJobInfo.setStatus(JobConf.getStatusString(finalStreamTask.getStatus())); StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); streamTaskService.updateTask(finalStreamTask); LOG.info("Streamis Job {} has added a new task successfully.", jobName); From f4e3cf5bf3a50daa619b4637ae37815d891d1c2b Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 20 Oct 2022 22:28:24 +0800 Subject: [PATCH 013/219] The strategy to retry the request and compact the buffer in log collector. --- .../log/collector/ExceptionListener.java | 15 ++++ .../sender/AbstractRpcLogSender.java | 25 ++++-- .../log/collector/sender/RpcLogSender.java | 6 ++ .../log/collector/sender/RpcSenderConfig.java | 1 + .../sender/buf/ImmutableSendBuffer.java | 8 +- .../sender/http/AbstractHttpLogSender.java | 80 ++++++++++++++++--- .../jobmanager/log/entities/LogElement.java | 6 ++ .../log/entities/StreamisLogEvents.java | 5 ++ 8 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java new file mode 100644 index 000000000..4c9ac6ea8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +/** + * Exception listener + */ +public interface ExceptionListener { + + /** + * Listen the exception + * @param subject the subject that throws the exception + * @param t Throwable + * @param message message + */ + void onException(Object subject, Throwable t, String message); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index 6b4d36580..a30ff98c6 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.ExceptionListener; import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.ImmutableSendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; @@ -42,7 +43,11 @@ public abstract class AbstractRpcLogSender implements R */ private volatile RpcLogContext rpcLogContext; - + protected boolean isTerminated = false; + /** + * Use the listener instead of log4j structure + */ + protected ExceptionListener exceptionListener; public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize){ this(rpcSenderConfig, cacheSize, sendBufSize, Integer.MAX_VALUE); } @@ -59,8 +64,7 @@ public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int @Override public LogCache getOrCreateLogCache() { - getOrCreateRpcLogContext(); - return null; + return getOrCreateRpcLogContext().getLogCache(); } @Override @@ -69,7 +73,9 @@ public void sendLog(T log) { try { getOrCreateLogCache().cacheLog(log); } catch (InterruptedException e) { - // Exception handler + // Invoke exception listener + Optional.ofNullable(exceptionListener).ifPresent(listener -> + listener.onException(this, e, null)); } } @@ -78,6 +84,11 @@ public void syncSendLog(T log) { } + @Override + public void setExceptionListener(ExceptionListener listener) { + this.exceptionListener = listener; + } + @Override public void close() { @@ -95,7 +106,7 @@ public void close() { * @param aggregatedEntity agg entity * @param rpcSenderConfig rpc sender config */ - protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig); + protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws Exception; /** * Send log exception strategy @@ -302,7 +313,7 @@ public int drainLogsTo(List elements, int maxElements) { return 0; } - // Equal to the poll method in ArrayBlockingQuue + // Equal to the poll method in ArrayBlockingQueue @Override public T takeLog(long timeout, TimeUnit unit) throws InterruptedException { long nanos = unit.toNanos(timeout); @@ -410,6 +421,8 @@ private int sendBuf(SendBuffer sendBuffer, Object[] items, int takeIndex, int int send = sendBuffer.writeBuf((T[]) items, takeIndex, len); if (send < len){ // Buffer full exception + exceptionListener.onException(this, null, "The sender buffer is full," + + " expected: [" + len + "], actual: [" + send + "]"); } // Allow data loss return send; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java index 25f512643..8254f0a34 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.ExceptionListener; import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; @@ -26,6 +27,11 @@ public interface RpcLogSender { */ void syncSendLog(T log); + /** + * Exception listener + * @param listener listener + */ + void setExceptionListener(ExceptionListener listener); /** * Close sender */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java index d7a307f2a..aac1a9578 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java @@ -64,6 +64,7 @@ public class RpcSenderConfig { * The time for server recovery */ private int serverRecoveryTimeInSec = 5; + /** * Retry max delay time of sender */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index eefe50ae7..3c61985f6 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -1,5 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; + import java.util.Arrays; import java.util.function.Function; @@ -30,14 +32,13 @@ public void capacity(String newCapacity) { } @Override + @SuppressWarnings("all") public int writeBuf(E[] elements, int srcIndex, int length) { if (srcIndex < elements.length){ int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); if (startPos >= 0){ int writes = position() - startPos; - for (int i = srcIndex; i < writes; i ++){ - buf[startPos++] = elements[i]; - } + System.arraycopy(this.buf, startPos, elements, srcIndex, writes); return writes; } } @@ -68,4 +69,5 @@ public E readBuf() { public SendBuffer compact(Function dropAble) { return null; } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index 7f2c68d04..bebfd64b7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -1,34 +1,57 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.http.HttpException; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; +import org.apache.http.conn.ConnectTimeoutException; +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.UnknownHostException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; public abstract class AbstractHttpLogSender extends AbstractRpcLogSender { /** * Retry strategy */ - private SendLogExceptionStrategy sendRetryStrategy; + private final SendLogExceptionStrategy sendRetryStrategy; /** * Exception counter */ - private AtomicInteger exceptionCounter = new AtomicInteger(); + private final AtomicInteger exceptionCounter = new AtomicInteger(); /** * Hold the global http client */ - private HttpClient globalHttpClient; + private final HttpClient globalHttpClient; + + /** + * Recover time point + */ + private final AtomicLong serverRecoveryTimePoint = new AtomicLong(-1L); public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { super(rpcSenderConfig, cacheSize, sendBufSize); this.globalHttpClient = HttpClientTool.createHttpClient(rpcSenderConfig); + final RpcLogSender sender = this; this.sendRetryStrategy = new SendLogExceptionStrategy(this) { + + private final Class[] retryOnExceptions = new Class[]{ + InterruptedIOException.class, UnknownHostException.class, + ConnectTimeoutException.class, SSLException.class}; @Override public int retryCount() { return rpcSenderConfig.getSendRetryCnt(); @@ -36,21 +59,54 @@ public int retryCount() { @Override public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { - - return null; + boolean shouldRetry = false; + // Limit of exception number is the same as the retry times + if (exceptionCounter.incrementAndGet() > retryCount()){ + serverRecoveryTimePoint.set(System.currentTimeMillis() + + TimeUnit.SECONDS.toMillis(rpcSenderConfig.getServerRecoveryTimeInSec())); + } else { + for (Class retryOnException : retryOnExceptions) { + if (retryOnException.isAssignableFrom(e.getClass())) { + shouldRetry = true; + break; + } + } + } + if (shouldRetry && !sender.getOrCreateLogCache().isCacheable()){ + // Means that the cache is full + // Set the position of buffer to 0 + sendBuffer.rewind(); + // Compact the buffer and transient to write mode; + sendBuffer.compact( element -> element.mark() > 1); + shouldRetry = false; + } + exceptionListener.onException(sender, e, null); + return new RetryDescription(shouldRetry); } }; } @Override - protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) { - if (aggregatedEntity instanceof LogElement){ - long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); - if (System.currentTimeMillis() - timestamp > rpcSenderConfig.getMaxDelayTimeInSec() * 1000L){ - // Abort the entity - return; + protected SendLogExceptionStrategy getSendLogExceptionStrategy() { + return this.sendRetryStrategy; + } + + @Override + protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws IOException { + if (System.currentTimeMillis() >= serverRecoveryTimePoint.get()) { + if (aggregatedEntity instanceof LogElement) { + long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); + if (System.currentTimeMillis() - timestamp > rpcSenderConfig.getMaxDelayTimeInSec() * 1000L) { + // Abort the entity + return; + } } + EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); + postAction.getRequestHeaders().put(rpcSenderConfig.getTokenUserKey(), rpcSenderConfig.getTokenUser()); + // Ignore the response + postAction.execute(this.globalHttpClient); + // Init the counter + this.exceptionCounter.set(0); } - } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index 8a497b61e..b0694580c 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -25,4 +25,10 @@ public interface LogElement { */ String[] getContents(); + /** + * The importance of log + * 0: useless, 1: normal, 2:important + * @return + */ + int mark(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index f194fb289..3a1766ee6 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -16,4 +16,9 @@ public long getLogTimeStamp() { public String[] getContents() { return new String[0]; } + + @Override + public int mark() { + return 0; + } } From 6153eb956bac9e18347a1fe5327627d340ae965c Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 21 Oct 2022 03:26:35 +0800 Subject: [PATCH 014/219] Complete the log collector (95%) --- .../streamis-job-log-collector/pom.xml | 33 ++- .../collector/StreamisLogAppenderConfig.java | 4 - .../log/collector/StreamisRpcLogAppender.java | 97 ++++++++- .../log/collector/config/RpcAuthConfig.java | 92 +++++++++ .../collector/config/RpcLogSenderConfig.java | 172 ++++++++++++++++ .../collector/config/SendBufferConfig.java | 53 +++++ .../collector/config/SendLogCacheConfig.java | 55 +++++ .../config/StreamisLogAppenderConfig.java | 49 +++++ .../StreamisLogAppenderConfigBuilder.java | 24 +++ .../sender/AbstractRpcLogSender.java | 23 ++- .../log/collector/sender/RpcSenderConfig.java | 189 ------------------ .../sender/SendLogCacheConsumer.java | 11 +- .../sender/StreamisRpcLogSender.java | 33 ++- .../sender/buf/AbstractSendBuffer.java | 16 +- .../sender/buf/ImmutableSendBuffer.java | 39 +++- .../sender/http/AbstractHttpLogSender.java | 17 +- .../collector/sender/http/HttpClientTool.java | 11 +- .../log/collector/sender/http/Json.java | 1 + .../streamis-job-log-common/pom.xml | 9 + .../jobmanager/log/entities/LogElement.java | 1 - .../log/entities/StreamisLogEvent.java | 57 +++++- .../log/entities/StreamisLogEvents.java | 50 ++++- 22 files changed, 794 insertions(+), 242 deletions(-) delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index bbc6a69c0..5fd394795 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -16,7 +16,8 @@ 8 4.5.13 4.5.4 - 2.13.2.2 + 2.17.1 + 1.7.15 @@ -36,11 +37,33 @@ httpmime ${httpmine.version} - + - com.fasterxml.jackson.core - jackson-databind - ${jackson-databind.version} + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + provided + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + provided + + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + provided \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java deleted file mode 100644 index 5c0ff730d..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector; - -public class StreamisLogAppenderConfig { -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index d326e6e04..a13de8d83 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -1,5 +1,100 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector; -public class StreamisRpcLogAppender { +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfigBuilder; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.layout.PatternLayout; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.ServiceLoader; +import java.util.concurrent.TimeUnit; + +/** + * Streamis rpc log appender + */ +@Plugin(name = "StreamRpcLog", category = "Core", elementType = "appender", printObject = true) +public class StreamisRpcLogAppender extends AbstractAppender { + + private static final String DEFAULT_APPENDER_NAME = "StreamRpcLog"; + + /** + * Appender config + */ + private StreamisLogAppenderConfig appenderConfig; + + /** + * Rpc log sender + */ + private StreamisRpcLogSender rpcLogSender; + + /** + * Cache + */ + private LogCache logCache; + protected StreamisRpcLogAppender(String name, Filter filter, + Layout layout, + boolean ignoreExceptions, Property[] properties, + StreamisLogAppenderConfig appenderConfig) { + super(name, filter, layout, ignoreExceptions, properties); + this.appenderConfig = appenderConfig; + this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), + this.appenderConfig.getSenderConfig()); + this.logCache = this.rpcLogSender.getOrCreateLogCache(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> this.rpcLogSender.close())); + } + + @Override + public void append(LogEvent event) { + String content = new String(getLayout().toByteArray(event)); + // Transform to stream log event; + StreamisLogEvent logEvent = new StreamisLogEvent(content, System.currentTimeMillis()); + try { + this.logCache.cacheLog(logEvent); + } catch (InterruptedException e) { + LOGGER.error("StreamisRpcLogAppender: {} interrupted when cache the log into the RPC sender, message: {}", this.getName(), e.getMessage()); + } + } + + @PluginFactory + public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") String name, + @PluginAttribute("appName") String applicationName, + @PluginAttribute("ignoreExceptions") boolean ignoreExceptions, + @PluginElement("Filter") final Filter filter, + @PluginElement("Layout") Layout layout, + @PluginElement("RpcLogSender")RpcLogSenderConfig rpcLogSenderConfig){ + if (null == name || name.trim().equals("")){ + name = DEFAULT_APPENDER_NAME; + } + if (Objects.isNull(layout)){ + layout = PatternLayout.createDefaultLayout(); + } + // Search the config builder + List configBuilders = new ArrayList<>(); + StreamisLogAppenderConfig logAppenderConfig = null; + ServiceLoader.load(StreamisLogAppenderConfigBuilder.class, + StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configBuilders::add); + if (!configBuilders.isEmpty()){ + logAppenderConfig = configBuilders.get(0).build(applicationName, filter, rpcLogSenderConfig); + } + if (Objects.isNull(logAppenderConfig)){ + logAppenderConfig = new StreamisLogAppenderConfig(applicationName, filter, rpcLogSenderConfig); + } + return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java new file mode 100644 index 000000000..e88eb91e9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java @@ -0,0 +1,92 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; + +/** + * Authentication config + */ +@Plugin( + name = "AuthConfig", + category = "Core", + printObject = true +) +public class RpcAuthConfig { + /** + * Key of token-code + */ + private String tokenCodeKey = "Token-Code"; + + /** + * Key of token-user + */ + private String tokenUserKey = "Token-User"; + + /** + * Token user + */ + private String tokenUser = System.getProperty("user.name"); + + /** + * Token code + */ + private String tokenCode = "STREAM-LOG"; + + public RpcAuthConfig(){ + + } + + public RpcAuthConfig(String tokenCodeKey, String tokenCode, String tokenUserKey, String tokenUser){ + if (null != tokenCodeKey) { + this.tokenCodeKey = tokenCodeKey; + } + if (null != tokenCode){ + this.tokenCode = tokenCode; + } + if (null != tokenUserKey){ + this.tokenUserKey = tokenUserKey; + } + if (null != tokenUser){ + this.tokenUser = tokenUser; + } + } + + @PluginFactory + public static RpcAuthConfig createRpcAuthConfig(@PluginAttribute("tokenCodeKey") String tokenCodeKey, + @PluginAttribute("tokenCode") String tokenCode, + @PluginAttribute("tokenUserKey") String tokenUserKey, @PluginAttribute("tokenUser") String tokenUser){ + return new RpcAuthConfig(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); + } + public String getTokenCodeKey() { + return tokenCodeKey; + } + + public void setTokenCodeKey(String tokenCodeKey) { + this.tokenCodeKey = tokenCodeKey; + } + + public String getTokenUserKey() { + return tokenUserKey; + } + + public void setTokenUserKey(String tokenUserKey) { + this.tokenUserKey = tokenUserKey; + } + + public String getTokenUser() { + return tokenUser; + } + + public void setTokenUser(String tokenUser) { + this.tokenUser = tokenUser; + } + + public String getTokenCode() { + return tokenCode; + } + + public void setTokenCode(String tokenCode) { + this.tokenCode = tokenCode; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java new file mode 100644 index 000000000..10a063946 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java @@ -0,0 +1,172 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +import java.util.Objects; + +/** + * Rpc sender configuration + */ +@Plugin( + name = "RpcLogSender", + category = "Core", + printObject = true +) +public class RpcLogSenderConfig { + + /** + * Send address + */ + private String address; + + /** + * Timeout of connecting + */ + private int connectionTimeout = 3000; + + /** + * Timeout of reading from socket + */ + private int socketTimeout = 15000; + + /** + * Retry count of sending + */ + private int sendRetryCnt = 3; + + /** + * The time for server recovery + */ + private int serverRecoveryTimeInSec = 5; + + /** + * Retry max delay time of sender + */ + private int maxDelayTimeInSec = 60; + + /** + * Auth config + */ + private RpcAuthConfig authConfig = new RpcAuthConfig(); + + /** + * Cache config + */ + private SendLogCacheConfig cacheConfig = new SendLogCacheConfig(); + + /** + * Buffer config + */ + private SendBufferConfig bufferConfig = new SendBufferConfig(); + + public RpcLogSenderConfig(){ + + } + + public RpcLogSenderConfig(String address, int sendRetryCnt, int connectionTimeout, int socketTimeout, + int serverRecoveryTimeInSec, int maxDelayTimeInSec, + RpcAuthConfig authConfig, SendLogCacheConfig cacheConfig, SendBufferConfig bufferConfig){ + this.address = address; + this.sendRetryCnt = sendRetryCnt; + this.connectionTimeout = connectionTimeout; + this.socketTimeout = socketTimeout; + this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; + this.maxDelayTimeInSec = maxDelayTimeInSec; + if (Objects.nonNull(authConfig)){ + this.authConfig = authConfig; + } + if (Objects.nonNull(cacheConfig)){ + this.cacheConfig = cacheConfig; + } + if (Objects.nonNull(bufferConfig)){ + this.bufferConfig = bufferConfig; + } + } + @PluginFactory + public static RpcLogSenderConfig createConfig( + @PluginAttribute("address") String address, @PluginAttribute("sendRetryCnt") String sendRetryCnt, + @PluginAttribute("connectionTimeout") String connectionTimeout, @PluginAttribute("socketTimeout") String socketTimeout, + @PluginAttribute("serverRecoveryTimeInSec") String serverRecoveryTimeInSec, @PluginAttribute("maxDelayTimeInSec") String maxDelayTimeInSec, + @PluginElement("AuthConfig")RpcAuthConfig authConfig, @PluginElement("SendLogCache") SendLogCacheConfig cacheConfig, + @PluginElement("SendBuffer")SendBufferConfig bufferConfig){ + return new RpcLogSenderConfig(address, Integers.parseInt(sendRetryCnt, 3), + Integers.parseInt(connectionTimeout, 3000), Integers.parseInt(socketTimeout, 15000), + Integers.parseInt(serverRecoveryTimeInSec, 5), Integers.parseInt(maxDelayTimeInSec, 60), + authConfig, cacheConfig, bufferConfig); + } + public RpcAuthConfig getAuthConfig() { + return authConfig; + } + + public void setAuthConfig(RpcAuthConfig authConfig) { + this.authConfig = authConfig; + } + + public SendLogCacheConfig getCacheConfig() { + return cacheConfig; + } + + public void setCacheConfig(SendLogCacheConfig cacheConfig) { + this.cacheConfig = cacheConfig; + } + + public SendBufferConfig getBufferConfig() { + return bufferConfig; + } + + public void setBufferConfig(SendBufferConfig bufferConfig) { + this.bufferConfig = bufferConfig; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public int getSendRetryCnt() { + return sendRetryCnt; + } + + public void setSendRetryCnt(int sendRetryCnt) { + this.sendRetryCnt = sendRetryCnt; + } + + public int getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public int getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public int getMaxDelayTimeInSec() { + return maxDelayTimeInSec; + } + + public void setMaxDelayTimeInSec(int maxDelayTimeInSec) { + this.maxDelayTimeInSec = maxDelayTimeInSec; + } + + public int getServerRecoveryTimeInSec() { + return serverRecoveryTimeInSec; + } + + public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { + this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java new file mode 100644 index 000000000..651caa616 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +@Plugin( + name = "SendBuffer", + category = "Core", + printObject = true +) +public class SendBufferConfig { + /** + * Size of send buffer + */ + private int size = 50; + + /** + * Expire time of send buffer + */ + private long expireTimeInSec = 2; + + public SendBufferConfig(){ + + } + + public SendBufferConfig(int size, long expireTimeInSec){ + this.size = size; + this.expireTimeInSec = expireTimeInSec; + } + + @PluginFactory + public static SendBufferConfig createBufferConfig( + @PluginAttribute("size") String size, @PluginAttribute("expireTimeInSec") String expireTimeInSec){ + return new SendBufferConfig(Integers.parseInt(size, 50), Integers.parseInt(expireTimeInSec, 2)); + } + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public long getExpireTimeInSec() { + return expireTimeInSec; + } + + public void setExpireTimeInSec(long expireTimeInSec) { + this.expireTimeInSec = expireTimeInSec; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java new file mode 100644 index 000000000..6921584a5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +/** + * Cache config + */ +@Plugin( + name = "SendLogCache", + category = "Core", + printObject = true +) +public class SendLogCacheConfig { + /** + * Size of send cache + */ + private int size = 150; + + /** + * Max number of consuming thread + */ + private int maxConsumeThread = 10; + + public SendLogCacheConfig(){ + + } + + public SendLogCacheConfig(int size, int maxConsumeThread){ + this.size = size; + this.maxConsumeThread = maxConsumeThread; + } + @PluginFactory + public static SendLogCacheConfig createCacheConfig( + @PluginAttribute("size") String size, @PluginAttribute("maxConsumeThread") String maxConsumeThread){ + return new SendLogCacheConfig(Integers.parseInt(size, 150), Integers.parseInt(maxConsumeThread, 10)); + } + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getMaxConsumeThread() { + return maxConsumeThread; + } + + public void setMaxConsumeThread(int maxConsumeThread) { + this.maxConsumeThread = maxConsumeThread; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java new file mode 100644 index 000000000..40a6a7d0d --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.Filter; + +/** + * Appender config + */ +public class StreamisLogAppenderConfig { + + private String applicationName; + + /** + * Filter in log4j + */ + private Filter filter; + + private RpcLogSenderConfig senderConfig; + + public StreamisLogAppenderConfig(String applicationName, Filter filter, + RpcLogSenderConfig rpcLogSenderConfig){ + this.applicationName = applicationName; + this.filter = filter; + this.senderConfig = null != rpcLogSenderConfig? rpcLogSenderConfig : new RpcLogSenderConfig(); + } + + public String getApplicationName() { + return applicationName; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public Filter getFilter() { + return filter; + } + + public void setFilter(Filter filter) { + this.filter = filter; + } + + public RpcLogSenderConfig getSenderConfig() { + return senderConfig; + } + + public void setSenderConfig(RpcLogSenderConfig senderConfig) { + this.senderConfig = senderConfig; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java new file mode 100644 index 000000000..cecc589a1 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.Filter; + +import java.util.Map; + +/** + * Config builder for appender + */ +public abstract class StreamisLogAppenderConfigBuilder { + + + /** + * Build method + * @param applicationName application name + * @param filter filter + * @param senderConfig sender config + * @return + */ + public abstract StreamisLogAppenderConfig build(String applicationName, + Filter filter, RpcLogSenderConfig senderConfig); + + abstract Map loadConfigProps(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index a30ff98c6..5aa1bdcc4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -2,6 +2,8 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.ExceptionListener; import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.SendLogCacheConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.ImmutableSendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; @@ -36,7 +38,7 @@ public abstract class AbstractRpcLogSender implements R /** * Connect config */ - protected RpcSenderConfig rpcSenderConfig; + protected RpcLogSenderConfig rpcSenderConfig; /** * Rpc log context @@ -48,18 +50,18 @@ public abstract class AbstractRpcLogSender implements R * Use the listener instead of log4j structure */ protected ExceptionListener exceptionListener; - public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize){ - this(rpcSenderConfig, cacheSize, sendBufSize, Integer.MAX_VALUE); - } - public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize, int maxCacheConsume){ + public AbstractRpcLogSender(RpcLogSenderConfig rpcSenderConfig){ this.rpcSenderConfig = rpcSenderConfig; + SendLogCacheConfig cacheConfig = rpcSenderConfig.getCacheConfig(); + this.cacheSize = cacheConfig.getSize(); + this.maxCacheConsume = cacheConfig.getMaxConsumeThread(); + this.sendBufSize = rpcSenderConfig.getBufferConfig().getSize(); + if (sendBufSize > cacheSize) { throw new IllegalArgumentException("Size of send buffer is larger than cache size"); } - this.cacheSize = cacheSize; - this.sendBufSize = sendBufSize; - this.maxCacheConsume = maxCacheConsume; + } @Override @@ -91,7 +93,8 @@ public void setExceptionListener(ExceptionListener listener) { @Override public void close() { - + getOrCreateRpcLogContext().destroyCacheConsumers(); + this.isTerminated = true; } /** @@ -106,7 +109,7 @@ public void close() { * @param aggregatedEntity agg entity * @param rpcSenderConfig rpc sender config */ - protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws Exception; + protected abstract void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) throws Exception; /** * Send log exception strategy diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java deleted file mode 100644 index aac1a9578..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java +++ /dev/null @@ -1,189 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; - -/** - * Rpc sender configuration - */ -public class RpcSenderConfig { - - /** - * Send address - */ - private String address; - - /** - * Key of token-code - */ - private String tokenCodeKey = "Token-Code"; - - /** - * Key of token-user - */ - private String tokenUserKey = "Token-User"; - - /** - * Token user - */ - private String tokenUser = System.getProperty("user.name"); - - /** - * Token code - */ - private String tokenCode = "STREAM_LOG"; - - /** - * Timeout of connecting - */ - private int connectionTimeout = 3000; - - /** - * Timeout of reading from socket - */ - private int socketTimeout = 15000; - - /** - * Size of send cache - */ - private int cacheSize = 150; - - /** - * Size of send buffer - */ - private int bufferSize = 50; - - /** - * Expire time of send buffer - */ - private long bufferExpireTimeInSec = -1; - - /** - * Retry count of sending - */ - private int sendRetryCnt = 3; - - /** - * The time for server recovery - */ - private int serverRecoveryTimeInSec = 5; - - /** - * Retry max delay time of sender - */ - private int maxDelayTimeInSec; - - /** - * Max number of consuming thread - */ - private int maxConsumeThread = 10; - - public long getBufferExpireTimeInSec() { - return bufferExpireTimeInSec; - } - - public void setBufferExpireTimeInSec(long bufferExpireTimeInSec) { - this.bufferExpireTimeInSec = bufferExpireTimeInSec; - } - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - public int getCacheSize() { - return cacheSize; - } - - public void setCacheSize(int cacheSize) { - this.cacheSize = cacheSize; - } - - public int getBufferSize() { - return bufferSize; - } - - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - - public int getSendRetryCnt() { - return sendRetryCnt; - } - - public void setSendRetryCnt(int sendRetryCnt) { - this.sendRetryCnt = sendRetryCnt; - } - - public int getConnectionTimeout() { - return connectionTimeout; - } - - public void setConnectionTimeout(int connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public int getSocketTimeout() { - return socketTimeout; - } - - public void setSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - public int getMaxConsumeThread() { - return maxConsumeThread; - } - - public void setMaxConsumeThread(int maxConsumeThread) { - this.maxConsumeThread = maxConsumeThread; - } - - public String getTokenCodeKey() { - return tokenCodeKey; - } - - public void setTokenCodeKey(String tokenCodeKey) { - this.tokenCodeKey = tokenCodeKey; - } - - public String getTokenCode() { - return tokenCode; - } - - public void setTokenCode(String tokenCode) { - this.tokenCode = tokenCode; - } - - public int getMaxDelayTimeInSec() { - return maxDelayTimeInSec; - } - - public void setMaxDelayTimeInSec(int maxDelayTimeInSec) { - this.maxDelayTimeInSec = maxDelayTimeInSec; - } - - public String getTokenUserKey() { - return tokenUserKey; - } - - public void setTokenUserKey(String tokenUserKey) { - this.tokenUserKey = tokenUserKey; - } - - public String getTokenUser() { - return tokenUser; - } - - public void setTokenUser(String tokenUser) { - this.tokenUser = tokenUser; - } - - public int getServerRecoveryTimeInSec() { - return serverRecoveryTimeInSec; - } - - public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { - this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; - } -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index e851943a7..8a3a05a61 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; @@ -38,12 +39,13 @@ public abstract class SendLogCacheConsumer implements Runn public SendLogCacheConsumer(String id, SendLogCache cache, SendBuffer sendBuffer, - RpcSenderConfig rpcSenderConfig){ + RpcLogSenderConfig rpcSenderConfig){ this.id = id; this.cache = cache; this.sendBuffer = sendBuffer; - this.bufferExpireTimeInMills = rpcSenderConfig.getBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS - .toMillis(rpcSenderConfig.getBufferExpireTimeInSec()) : -1; + long expireTimeInSec = rpcSenderConfig.getBufferConfig().getExpireTimeInSec(); + this.bufferExpireTimeInMills = expireTimeInSec > 0 ? TimeUnit.SECONDS + .toMillis(expireTimeInSec) : -1; } @@ -93,6 +95,9 @@ public void run() { public void shutdown(){ this.isTerminated = true; + if (null != this.future){ + this.future.cancel(true); + } } public Future getFuture() { diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java index 06d70407a..02348c3af 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -1,6 +1,37 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.AbstractHttpLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; -public class StreamisRpcLogSender{ +/** + * Log sender for streamis + */ +public class StreamisRpcLogSender extends AbstractHttpLogSender { + + /** + * Each sender register an application + */ + private final String applicationName; + + public StreamisRpcLogSender(String applicationName, RpcLogSenderConfig rpcSenderConfig) { + super(rpcSenderConfig); + this.applicationName = applicationName; + } + + /** + * Aggregate to streamis log events + * @param sendBuffer send buffer + * @return + */ + @Override + protected StreamisLogEvents aggregateBuffer(SendBuffer sendBuffer) { + int remain = sendBuffer.remaining(); + StreamisLogEvent[] logEvents = new StreamisLogEvent[remain]; + sendBuffer.readBuf(logEvents, 0, logEvents.length); + return new StreamisLogEvents(applicationName, logEvents); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java index 83e04ec7d..138817053 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java @@ -55,6 +55,7 @@ public int remaining() { @Override public void flip() { + checkFlag(Flag.WRITE_MODE); this.limit = this.position; this.position = 0; this.accessFlag = Flag.READ_MODE; @@ -94,9 +95,7 @@ final void limit(int newLimit){ * @return the current position value */ final int nextPosition(int offset, Flag accessFlag){ - if (this.accessFlag != accessFlag){ - throw new IllegalStateException("Illegal access flag [" + accessFlag + "] for send buffer"); - } + checkFlag(accessFlag); int p = position; // Reach the limit, return -1 value if (p >= limit){ @@ -108,6 +107,14 @@ final int nextPosition(int offset, Flag accessFlag){ return p; } + final void checkFlag(Flag accessFlag){ + if (this.accessFlag != accessFlag){ + throw new IllegalStateException("Illegal access flag [" + accessFlag + "] for send buffer"); + } + } + final void setFlag(Flag accessFlag){ + this.accessFlag = accessFlag; + } /** * * @return the current position @@ -116,6 +123,9 @@ final int position(){ return this.position; } + final void position(int position){ + this.position = position; + } /** * Do the actual clear */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index 3c61985f6..4de1e6753 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -1,7 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; -import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; - import java.util.Arrays; import java.util.function.Function; @@ -38,7 +36,7 @@ public int writeBuf(E[] elements, int srcIndex, int length) { int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); if (startPos >= 0){ int writes = position() - startPos; - System.arraycopy(this.buf, startPos, elements, srcIndex, writes); + System.arraycopy(elements, srcIndex, this.buf, startPos, writes); return writes; } } @@ -46,8 +44,17 @@ public int writeBuf(E[] elements, int srcIndex, int length) { } @Override + @SuppressWarnings("all") public int readBuf(E[] elements, int srcIndex, int length) { - return 0; + if (srcIndex < elements.length){ + int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.READ_MODE); + if (startPos >= 0){ + int reads = position() - startPos; + System.arraycopy(this.buf, startPos, elements, srcIndex, reads); + return reads; + } + } + return -1; } @Override @@ -61,13 +68,35 @@ public int writeBuf(E element) { } @Override + @SuppressWarnings("unchecked") public E readBuf() { + int startPos = nextPosition(1, Flag.READ_MODE); + if (startPos >= 0){ + return (E)buf[startPos]; + } return null; } @Override + @SuppressWarnings("unchecked") public SendBuffer compact(Function dropAble) { - return null; + checkFlag(Flag.READ_MODE); + int offset = 0; + int compact = position() - 1; + for(int i = position(); i < capacity; i ++){ + Object element = buf[i]; + if (dropAble.apply((E)element)){ + buf[i] = null; + offset ++; + } else { + compact = i - offset; + buf[compact] = element; + } + } + position(compact + 1); + limit(this.capacity); + setFlag(Flag.WRITE_MODE); + return this; } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index bebfd64b7..187152728 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -1,16 +1,13 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcAuthConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcLogSender; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; import org.apache.http.conn.ConnectTimeoutException; @@ -43,10 +40,9 @@ public abstract class AbstractHttpLogSender extends Abs */ private final AtomicLong serverRecoveryTimePoint = new AtomicLong(-1L); - public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { - super(rpcSenderConfig, cacheSize, sendBufSize); + public AbstractHttpLogSender(RpcLogSenderConfig rpcSenderConfig) { + super(rpcSenderConfig); this.globalHttpClient = HttpClientTool.createHttpClient(rpcSenderConfig); - final RpcLogSender sender = this; this.sendRetryStrategy = new SendLogExceptionStrategy(this) { private final Class[] retryOnExceptions = new Class[]{ @@ -92,7 +88,7 @@ protected SendLogExceptionStrategy getSendLogExceptionStrategy() { } @Override - protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws IOException { + protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) throws IOException { if (System.currentTimeMillis() >= serverRecoveryTimePoint.get()) { if (aggregatedEntity instanceof LogElement) { long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); @@ -102,7 +98,8 @@ protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throw } } EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); - postAction.getRequestHeaders().put(rpcSenderConfig.getTokenUserKey(), rpcSenderConfig.getTokenUser()); + RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); + postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); // Ignore the response postAction.execute(this.globalHttpClient); // Init the counter diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java index df4eb31ea..e67bec6da 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import org.apache.http.Header; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; @@ -38,7 +38,7 @@ public class HttpClientTool { * @param rpcSenderConfig rpc sender config * @return http client */ - public static HttpClient createHttpClient(RpcSenderConfig rpcSenderConfig){ + public static HttpClient createHttpClient(RpcLogSenderConfig rpcSenderConfig){ int connectTimeout = rpcSenderConfig.getConnectionTimeout() > 0? rpcSenderConfig.getConnectionTimeout() : DEFAULT_CONNECT_TIMEOUT; int socketTimeout = rpcSenderConfig.getSocketTimeout() > 0? rpcSenderConfig.getSocketTimeout() : DEFAULT_SOCKET_TIMEOUT; RequestConfig requestConfig = RequestConfig.custom() @@ -46,12 +46,13 @@ public static HttpClient createHttpClient(RpcSenderConfig rpcSenderConfig){ .setConnectionRequestTimeout(socketTimeout) .setSocketTimeout(socketTimeout) .build(); - int maxConn = rpcSenderConfig.getMaxConsumeThread() > 0? rpcSenderConfig.getMaxConsumeThread() : DEFAULT_MAX_CONN; + int maxConsumeThread = rpcSenderConfig.getCacheConfig().getMaxConsumeThread(); + int maxConn = maxConsumeThread > 0? maxConsumeThread : DEFAULT_MAX_CONN; HttpClientBuilder clientBuilder = HttpClients.custom(); - String tokenValue = rpcSenderConfig.getTokenCode(); + String tokenValue = rpcSenderConfig.getAuthConfig().getTokenCode(); List
defaultHeaders = new ArrayList<>(); if (null != tokenValue && !tokenValue.trim().equals("")){ - defaultHeaders.add(new BasicHeader(rpcSenderConfig.getTokenCodeKey(), tokenValue)); + defaultHeaders.add(new BasicHeader(rpcSenderConfig.getAuthConfig().getTokenCodeKey(), tokenValue)); } clientBuilder.setDefaultRequestConfig(requestConfig).setDefaultHeaders(defaultHeaders) .useSystemProperties().setMaxConnTotal(maxConn); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java index 8c493d042..40b20b50e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java @@ -126,5 +126,6 @@ public static T convert(Object simpleObj, JavaType javaType) { public static ObjectMapper getMapper(){ return mapper; } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index 51d283f53..a675aa404 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -14,6 +14,15 @@ 8 8 + 2.13.2.2 + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index b0694580c..e253d2a32 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -import java.util.concurrent.TimeUnit; /** * Element defined of log diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index 0c73d83d2..5c82e22cb 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -1,4 +1,59 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -public class StreamisLogEvent { +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.io.Serializable; + +/** + * Log event for streamis + */ +public class StreamisLogEvent implements LogElement, Serializable { + + /** + * Log time + */ + private final long logTimeInMills; + + /** + * Log content + */ + private final String content; + + /** + * Mark + */ + private int mark; + + public StreamisLogEvent(String content, long logTimeInMills){ + this.content = content; + this.logTimeInMills = logTimeInMills; + } + @Override + public int getSequenceId() { + return 0; + } + + @Override + public long getLogTimeStamp() { + return this.logTimeInMills; + } + + @Override + @JsonIgnore + public String[] getContents() { + return new String[]{content}; + } + + public String getContent() { + return content; + } + + @Override + public int mark() { + return this.mark; + } + + public void mark(int mark){ + this.mark = mark; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 3a1766ee6..972d1a492 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -1,6 +1,35 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -public class StreamisLogEvents implements LogElement{ +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.io.Serializable; + +public class StreamisLogEvents implements LogElement, Serializable { + + /** + * Application name + */ + private String appName; + /** + * Log time + */ + private final long logTimeInMills; + + private final StreamisLogEvent[] events; + + public StreamisLogEvents(String applicationName, StreamisLogEvent[] events){ + this.appName = applicationName; + this.events = events; + long maxTime = -1; + for(StreamisLogEvent event : events){ + long time = event.getLogTimeStamp(); + if (time > maxTime){ + maxTime = time; + } + } + this.logTimeInMills = maxTime; + } + @Override public int getSequenceId() { return 0; @@ -8,17 +37,30 @@ public int getSequenceId() { @Override public long getLogTimeStamp() { - return 0; + return this.logTimeInMills; } @Override + @JsonIgnore public String[] getContents() { - return new String[0]; + String[] contents = new String[events.length]; + for(int i = 0 ; i < contents.length; i++){ + contents[i] = events[i].getContent(); + } + return contents; } @Override public int mark() { - return 0; + return 1; + } + + public String getAppName() { + return appName; + } + + public StreamisLogEvent[] getEvents() { + return events; } } From 7ac4336426115d58f2f45bf85f21bd93c2e54a92 Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 21 Oct 2022 10:49:32 +0800 Subject: [PATCH 015/219] Fix the problem in send buffer and consumer. --- .../log/collector/StreamisRpcLogAppender.java | 1 + .../StreamisLogAppenderConfigBuilder.java | 9 +++-- .../sender/AbstractRpcLogSender.java | 2 +- .../sender/SendLogCacheConsumer.java | 18 ++++++--- .../sender/StreamisRpcLogSender.java | 9 +++-- .../sender/buf/AbstractSendBuffer.java | 2 + .../sender/buf/ImmutableSendBuffer.java | 4 +- .../log/collector/sender/buf/SendBuffer.java | 4 +- .../sender/http/AbstractHttpLogSender.java | 7 +++- .../collector/StreamisLogAppenderTest.java | 16 ++++++++ .../src/test/resources/log4j2.xml | 37 +++++++++++++++++++ 11 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index a13de8d83..737885e1a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -18,6 +18,7 @@ import org.apache.logging.log4j.core.layout.PatternLayout; import java.io.Serializable; +import java.nio.Buffer; import java.util.ArrayList; import java.util.List; import java.util.Objects; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java index cecc589a1..f3374ac48 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java @@ -17,8 +17,11 @@ public abstract class StreamisLogAppenderConfigBuilder { * @param senderConfig sender config * @return */ - public abstract StreamisLogAppenderConfig build(String applicationName, - Filter filter, RpcLogSenderConfig senderConfig); + public StreamisLogAppenderConfig build(String applicationName, + Filter filter, RpcLogSenderConfig senderConfig){ + return null; + } - abstract Map loadConfigProps(); + + public abstract Map loadConfigProps(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index 5aa1bdcc4..d90eb1a7f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -421,7 +421,7 @@ public int drainLogsTo(SendBuffer sendBuffer, int maxElements) { @SuppressWarnings("unchecked") private int sendBuf(SendBuffer sendBuffer, Object[] items, int takeIndex, int len){ - int send = sendBuffer.writeBuf((T[]) items, takeIndex, len); + int send = sendBuffer.writeBuf(items, takeIndex, len); if (send < len){ // Buffer full exception exceptionListener.onException(this, null, "The sender buffer is full," + diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index 8a3a05a61..52a7b72cb 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -53,13 +53,16 @@ public SendLogCacheConsumer(String id, SendLogCache cache, public void run() { int remain; long expireTimeInMills = requireNewFlushTime(); + int capacity = sendBuffer.capacity(); while (!this.isTerminated) { try { - if ((expireTimeInMills > 0 && expireTimeInMills >= System.currentTimeMillis()) - || (remain = this.sendBuffer.remaining()) <= 0) { + remain = this.sendBuffer.remaining(); + if ((expireTimeInMills > 0 && expireTimeInMills <= System.currentTimeMillis()) || remain <= 0) { // Transient to the read mode - sendBuffer.flip(); - onFlushAndSend(sendBuffer); + if (remain < capacity) { + sendBuffer.flip(); + onFlushAndSend(sendBuffer); + } expireTimeInMills = requireNewFlushTime(); if (sendBuffer.isReadMode()) { // Clear the buffer and transient to the write mode, otherwise continue writing @@ -84,11 +87,16 @@ public void run() { if (this.isTerminated && e instanceof InterruptedException){ return; } else { - System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); + System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); // For the unknown exception clear the cache sendBuffer.clear(); expireTimeInMills = requireNewFlushTime(); } + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + // Ignore + } } } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java index 02348c3af..b9ec15537 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -30,8 +30,11 @@ public StreamisRpcLogSender(String applicationName, RpcLogSenderConfig rpcSender @Override protected StreamisLogEvents aggregateBuffer(SendBuffer sendBuffer) { int remain = sendBuffer.remaining(); - StreamisLogEvent[] logEvents = new StreamisLogEvent[remain]; - sendBuffer.readBuf(logEvents, 0, logEvents.length); - return new StreamisLogEvents(applicationName, logEvents); + if (remain > 0) { + StreamisLogEvent[] logEvents = new StreamisLogEvent[remain]; + sendBuffer.readBuf(logEvents, 0, logEvents.length); + return new StreamisLogEvents(applicationName, logEvents); + } + return null; } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java index 138817053..1b42ad957 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java @@ -103,6 +103,8 @@ final int nextPosition(int offset, Flag accessFlag){ } if (p + offset > limit){ this.position = limit; + } else { + this.position = p + offset; } return p; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index 4de1e6753..0e64c4ffa 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -31,7 +31,7 @@ public void capacity(String newCapacity) { @Override @SuppressWarnings("all") - public int writeBuf(E[] elements, int srcIndex, int length) { + public int writeBuf(Object[] elements, int srcIndex, int length) { if (srcIndex < elements.length){ int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); if (startPos >= 0){ @@ -45,7 +45,7 @@ public int writeBuf(E[] elements, int srcIndex, int length) { @Override @SuppressWarnings("all") - public int readBuf(E[] elements, int srcIndex, int length) { + public int readBuf(Object[] elements, int srcIndex, int length) { if (srcIndex < elements.length){ int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.READ_MODE); if (startPos >= 0){ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java index 28e3a6356..0a98580fb 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java @@ -64,7 +64,7 @@ public interface SendBuffer { * @param length the length to read * @return write num */ - int writeBuf(E[] elements, int srcIndex, int length); + int writeBuf(Object[] elements, int srcIndex, int length); /** * Read buffer element @@ -79,7 +79,7 @@ public interface SendBuffer { * @param length the length to write * @return read num */ - int readBuf(E[] elements, int srcIndex, int length); + int readBuf(Object[] elements, int srcIndex, int length); /** * Compact the buffer, avoid the useless elements diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index 187152728..d247ba4ad 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.net.UnknownHostException; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -55,6 +56,7 @@ public int retryCount() { @Override public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { + e.printStackTrace(); boolean shouldRetry = false; // Limit of exception number is the same as the retry times if (exceptionCounter.incrementAndGet() > retryCount()){ @@ -62,7 +64,7 @@ public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBu TimeUnit.SECONDS.toMillis(rpcSenderConfig.getServerRecoveryTimeInSec())); } else { for (Class retryOnException : retryOnExceptions) { - if (retryOnException.isAssignableFrom(e.getClass())) { + if (retryOnException.equals(e.getClass())) { shouldRetry = true; break; } @@ -76,7 +78,7 @@ public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBu sendBuffer.compact( element -> element.mark() > 1); shouldRetry = false; } - exceptionListener.onException(sender, e, null); + Optional.ofNullable(exceptionListener).ifPresent(listener -> listener.onException(sender, e, null)); return new RetryDescription(shouldRetry); } }; @@ -97,6 +99,7 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th return; } } + EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java new file mode 100644 index 000000000..189e573ae --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamisLogAppenderTest { + private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); + public static void main(String[] args) throws InterruptedException { + while(true){ + for(int i = 0; i < 100; i ++){ + LOG.info("Stream Log appender test"); + } + Thread.sleep(1000); + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml new file mode 100644 index 000000000..d806c5198 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + ` + + + + + + + + \ No newline at end of file From 9588e90a3f865d4d82aea62fb8660b39a6dcf57b Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Fri, 21 Oct 2022 12:38:08 +0800 Subject: [PATCH 016/219] Optimize the architecture of fetch Job details. --- .../service/DefaultStreamTaskService.scala | 31 ++++-- .../manager/service/StreamTaskService.scala | 6 +- .../manager/transform/TaskMetricsParser.scala | 18 ++++ .../parser/AbstractTaskMetricsParser.scala | 37 +++++++ .../parser/FlinkTaskMetricsParser.scala | 48 +++++++++ .../parser/SparkTaskMetricsParser.scala | 83 ++++++++++++++ .../jobmanager/restful/api/JobRestfulApi.java | 101 ++---------------- 7 files changed, 218 insertions(+), 106 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index 3bc49354f..d03b0b9e0 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -15,6 +15,10 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service +import java.util +import java.util.concurrent.Future +import java.util.{Calendar, function} + import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.dao.StreamJobConfMapper import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager @@ -27,17 +31,18 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.SpringContextHolder import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf.FLINK_JOB_STATUS_FAILED import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamTaskMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobProgressVo, JobStatusVo, PauseResultVo, ScheduleResultVo, StreamTaskListVo} -import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobErrorException, JobExecuteErrorException, JobFetchErrorException, JobPauseErrorException, JobTaskErrorException} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo._ +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamTask} +import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobExecuteErrorException, JobFetchErrorException, JobPauseErrorException, JobTaskErrorException} import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.FutureScheduler import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.AbstractStreamisSchedulerEvent.StreamisEventInfo -import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.{AbstractStreamisSchedulerEvent, StreamisPhaseInSchedulerEvent} +import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.StreamisPhaseInSchedulerEvent import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.StreamisPhaseInSchedulerEvent.ScheduleCommand import com.webank.wedatasphere.streamis.jobmanager.manager.transform.exception.TransformFailedErrorException -import com.webank.wedatasphere.streamis.jobmanager.manager.transform.{StreamisTransformJobBuilder, Transform} +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.{StreamisTransformJobBuilder, TaskMetricsParser, Transform} import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils +import javax.annotation.Resource import org.apache.commons.lang.StringUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.httpclient.dws.DWSHttpClient @@ -47,10 +52,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional -import java.util -import java.util.{Calendar, Date, function} -import java.util.concurrent.Future -import javax.annotation.Resource import scala.collection.JavaConverters._ @@ -60,6 +61,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ @Autowired private var streamTaskMapper:StreamTaskMapper=_ @Autowired private var streamJobMapper:StreamJobMapper=_ @Autowired private var streamisTransformJobBuilders: Array[StreamisTransformJobBuilder] = _ + @Autowired private var taskMetricsParser: Array[TaskMetricsParser] = _ @Resource private var jobLaunchManager: JobLaunchManager[_ <: JobInfo] = _ @@ -721,4 +723,15 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ } } + override def getJobDetailsVO(streamJob: StreamJob, version: String): JobDetailsVo = { + val flinkJobInfo = getTaskJobInfo(streamJob.getId, version) + val jobStateInfos = flinkJobInfo.getJobStates + val metricsStr = if (JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES.getValue.contains(streamJob.getJobType)) null + else if(jobStateInfos == null || jobStateInfos.length == 0) null + else jobStateInfos(0).getLocation + taskMetricsParser.find(_.canParse(streamJob)).map(_.parse(metricsStr)).filter { jobDetailsVO => + jobDetailsVO.setLinkisJobInfo(flinkJobInfo) + true + }.getOrElse(throw new JobFetchErrorException(30030, s"Cannot find a TaskMetricsParser to parse job details.")) + } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala index 65c584804..26591d23f 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala @@ -18,8 +18,8 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobState import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobProgressVo, JobStatusVo, PauseResultVo, StreamTaskListVo} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamTask} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobDetailsVo, JobProgressVo, JobStatusVo, PauseResultVo, StreamTaskListVo} import java.util import java.util.concurrent.Future /** @@ -154,4 +154,6 @@ trait StreamTaskService { def getStateInfo(taskId: Long): JobState + def getJobDetailsVO(streamJob: StreamJob, version: String): JobDetailsVo + } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala new file mode 100644 index 000000000..1e7ad7b68 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +trait TaskMetricsParser { + + def canParse(streamJob: StreamJob): Boolean + + def parse(metrics: String): JobDetailsVo + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala new file mode 100644 index 000000000..eb0de6d6e --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.TaskMetricsParser +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.httpclient.dws.DWSHttpClient + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +trait AbstractTaskMetricsParser extends TaskMetricsParser { + + override def parse(metrics: String): JobDetailsVo = { + val jobDetailsVO = new JobDetailsVo + val dataNumberDTOS = new util.ArrayList[JobDetailsVo.DataNumberDTO] + val loadConditionDTOs = new util.ArrayList[JobDetailsVo.LoadConditionDTO] + val realTimeTrafficDTOS = new util.ArrayList[JobDetailsVo.RealTimeTrafficDTO] + jobDetailsVO.setDataNumber(dataNumberDTOS) + jobDetailsVO.setLoadCondition(loadConditionDTOs) + jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS) + val metricsMap = if(StringUtils.isNotBlank(metrics)) DWSHttpClient.jacksonJson.readValue(metrics, classOf[util.Map[String, Object]]) + else new util.HashMap[String, Object](0) + parse(metricsMap, dataNumberDTOS, loadConditionDTOs, realTimeTrafficDTOS) + jobDetailsVO + } + + protected def parse(metricsMap: util.Map[String, Object], + dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], + loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], + realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala new file mode 100644 index 000000000..48eb8ad06 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo +import org.springframework.stereotype.Component + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +@Component +class FlinkTaskMetricsParser extends AbstractTaskMetricsParser { + + override def canParse(streamJob: StreamJob): Boolean = streamJob.getJobType.startsWith("flink.") + + override def parse(metricsMap: util.Map[String, Object], + dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], + loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], + realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit = { + // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. + val dataNumberDTO = new JobDetailsVo.DataNumberDTO + dataNumberDTO.setDataName("kafka topic") + dataNumberDTO.setDataNumber(109345) + dataNumberDTOS.add(dataNumberDTO) + + val loadConditionDTO = new JobDetailsVo.LoadConditionDTO + loadConditionDTO.setType("jobManager") + loadConditionDTO.setHost("localhost") + loadConditionDTO.setMemory("1.5") + loadConditionDTO.setTotalMemory("2.0") + loadConditionDTO.setGcLastTime("2020-08-01") + loadConditionDTO.setGcLastConsume("1") + loadConditionDTO.setGcTotalTime("2min") + loadConditionDTOs.add(loadConditionDTO) + + val realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO + realTimeTrafficDTO.setSourceKey("kafka topic") + realTimeTrafficDTO.setSourceSpeed("100 Records/S") + realTimeTrafficDTO.setTransformKey("transform") + realTimeTrafficDTO.setSinkKey("hbase key") + realTimeTrafficDTO.setSinkSpeed("10 Records/S") + realTimeTrafficDTOS.add(realTimeTrafficDTO) + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala new file mode 100644 index 000000000..392986b53 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo +import org.apache.linkis.common.utils.Utils +import org.springframework.stereotype.Component + +import scala.collection.JavaConverters._ + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +@Component +class SparkTaskMetricsParser extends AbstractTaskMetricsParser { + + override protected def parse(metricsMap: util.Map[String, Object], + dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], + loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], + realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit = { + val addDataNumberDTO: String => () = key => { + val batch = new JobDetailsVo.DataNumberDTO + batch.setDataName(key) + batch.setDataNumber(metricsMap.get(key).toString.toInt) + dataNumberDTOS.add(batch) + } + addDataNumberDTO("waitingBatchs") + addDataNumberDTO("runningBatchs") + addDataNumberDTO("completedBatchs") + metricsMap.get("executors") match { + case executors: util.List[util.Map[String, AnyRef]] if !executors.isEmpty => + executors.asScala.foreach { executor => + val loadConditionDTO = new JobDetailsVo.LoadConditionDTO + loadConditionDTO.setType(executor.get("type").asInstanceOf[String]) + loadConditionDTO.setHost(executor.get("host").asInstanceOf[String]) + loadConditionDTO.setMemory(executor.get("memory").asInstanceOf[String]) + loadConditionDTO.setTotalMemory(executor.get("totalMemory").asInstanceOf[String]) + loadConditionDTO.setGcLastTime(executor.get("gcLastTime").asInstanceOf[String]) + loadConditionDTO.setGcLastConsume(executor.get("gcLastConsume").asInstanceOf[String]) + loadConditionDTO.setGcTotalTime(executor.get("gcTotalTime").asInstanceOf[String]) + loadConditionDTOs.add(loadConditionDTO) + } + case _ => + val loadConditionDTO = new JobDetailsVo.LoadConditionDTO + loadConditionDTO.setType("Driver") + loadConditionDTO.setHost("") + loadConditionDTO.setMemory("") + loadConditionDTO.setTotalMemory("") + loadConditionDTO.setGcLastTime("") + loadConditionDTO.setGcLastConsume("") + loadConditionDTO.setGcTotalTime("") + loadConditionDTOs.add(loadConditionDTO) + } + val realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO + metricsMap.get("batchMetrics") match { + case batchMetrics: util.List[util.Map[String, Object]] if !batchMetrics.isEmpty => + val batchMetric = batchMetrics.asScala.maxBy(_.get("batchTime").asInstanceOf[String]) + realTimeTrafficDTO.setSourceKey(metricsMap.getOrDefault("source", "").asInstanceOf[String]) + realTimeTrafficDTO.setSourceSpeed(batchMetric.get("inputRecords") + " Records") + realTimeTrafficDTO.setTransformKey("processing") + realTimeTrafficDTO.setSinkKey(metricsMap.getOrDefault("sink", "").asInstanceOf[String]) + val sinkSpeed = if (batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) + Utils.msDurationToString(batchMetric.get("totalDelay").asInstanceOf[Long]) + " totalDelay" + else if (batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) + Utils.msDurationToString(batchMetric.get("taskExecuteTime").asInstanceOf[Long]) + " executeTime(Last Batch)" + else "" + realTimeTrafficDTO.setSinkSpeed(sinkSpeed) + case _ => + realTimeTrafficDTO.setSourceKey("") + realTimeTrafficDTO.setSourceSpeed(" Records/S") + realTimeTrafficDTO.setTransformKey("") + realTimeTrafficDTO.setSinkKey("") + realTimeTrafficDTO.setSinkSpeed(" Records/S") + } + realTimeTrafficDTOS.add(realTimeTrafficDTO) + } + + override def canParse(streamJob: StreamJob): Boolean = streamJob.getJobType.startsWith("spark.") +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 57d5473b5..acb1ac529 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -39,7 +39,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.linkis.common.utils.Utils; import org.apache.linkis.httpclient.dws.DWSHttpClient; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; @@ -198,104 +197,16 @@ public Message detailsJob(HttpServletRequest req, @RequestParam(value = "jobId", if (jobId == null) { JobExceptionManager.createException(30301, "jobId"); } - + String username = SecurityFilter.getLoginUsername(req); StreamJob streamJob = streamJobService.getJobById(jobId); + if (!streamJobService.hasPermission(streamJob, username) && + !this.privilegeService.hasAccessPrivilege(req, streamJob.getProjectName())) { + return Message.error("Have no permission to get Job details of StreamJob [" + jobId + "]"); + } if(streamJob == null) { return Message.error("not exists job " + jobId); } - FlinkJobInfo flinkJobInfo = streamTaskService.getTaskJobInfo(jobId,version); - JobDetailsVo jobDetailsVO = new JobDetailsVo(); - List dataNumberDTOS = new ArrayList<>(); - List loadConditionDTOs = new ArrayList<>(); - List realTimeTrafficDTOS = new ArrayList<>(); - JobStateInfo[] jobStateInfos = flinkJobInfo.getJobStates(); - if(JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType()) || - (jobStateInfos == null || jobStateInfos.length == 0)) { - // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. - JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); - dataNumberDTO.setDataName("kafka topic"); - dataNumberDTO.setDataNumber(109345); - dataNumberDTOS.add(dataNumberDTO); - - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType("jobManager"); - loadConditionDTO.setHost("localhost"); - loadConditionDTO.setMemory("1.5"); - loadConditionDTO.setTotalMemory("2.0"); - loadConditionDTO.setGcLastTime("2020-08-01"); - loadConditionDTO.setGcLastConsume("1"); - loadConditionDTO.setGcTotalTime("2min"); - loadConditionDTOs.add(loadConditionDTO); - - JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); - realTimeTrafficDTO.setSourceKey("kafka topic"); - realTimeTrafficDTO.setSourceSpeed("100 Records/S"); - realTimeTrafficDTO.setTransformKey("transform"); - realTimeTrafficDTO.setSinkKey("hbase key"); - realTimeTrafficDTO.setSinkSpeed("10 Records/S"); - realTimeTrafficDTOS.add(realTimeTrafficDTO); - - } else { - String metricsStr = jobStateInfos[0].getLocation(); - Map metricsMap = DWSHttpClient.jacksonJson().readValue(metricsStr, Map.class); - JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); - dataNumberDTO.setDataName("waitingBatchs"); - dataNumberDTO.setDataNumber(Math.toIntExact((Long) metricsMap.get("waitingBatchs"))); - dataNumberDTOS.add(dataNumberDTO); - List> executors = (List>) metricsMap.get("executors"); - if(executors != null && !executors.isEmpty()) { - executors.forEach(map -> { - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType((String) map.get("type")); - loadConditionDTO.setHost((String) map.get("host")); - loadConditionDTO.setMemory((String) map.get("memory")); - loadConditionDTO.setTotalMemory((String) map.get("totalMemory")); - loadConditionDTO.setGcLastTime((String) map.get("gcLastTime")); - loadConditionDTO.setGcLastConsume((String) map.get("gcLastConsume")); - loadConditionDTO.setGcTotalTime((String) map.get("gcTotalTime")); - loadConditionDTOs.add(loadConditionDTO); - }); - } else { - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType("sparkAppMaster"); - loadConditionDTO.setHost(""); - loadConditionDTO.setMemory(""); - loadConditionDTO.setTotalMemory(""); - loadConditionDTO.setGcLastTime(""); - loadConditionDTO.setGcLastConsume(""); - loadConditionDTO.setGcTotalTime(""); - loadConditionDTOs.add(loadConditionDTO); - } - JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); - List> batchMetrics = (List>) metricsMap.get("batchMetrics"); - if(batchMetrics != null && !batchMetrics.isEmpty()) { - batchMetrics.stream().max(Comparator.comparing(map -> String.valueOf(map.get("batchTime")))).ifPresent(batchMetric -> { - realTimeTrafficDTO.setSourceKey((String) metricsMap.getOrDefault("source", "")); - realTimeTrafficDTO.setSourceSpeed(batchMetric.get("inputRecords") + " Records"); - realTimeTrafficDTO.setTransformKey("processing"); - realTimeTrafficDTO.setSinkKey((String) metricsMap.getOrDefault("sink", "")); - String totalDelay = ""; - if(batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) { - totalDelay = Utils.msDurationToString((long) batchMetric.get("totalDelay")) + " totalDelay"; - } else if(batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) { - totalDelay = Utils.msDurationToString((long) batchMetric.get("taskExecuteTime")) + " executeTime(Last Batch)"; - } - realTimeTrafficDTO.setSinkSpeed(totalDelay); - }); - } else { - realTimeTrafficDTO.setSourceKey(""); - realTimeTrafficDTO.setSourceSpeed(" Records/S"); - realTimeTrafficDTO.setTransformKey(""); - realTimeTrafficDTO.setSinkKey(""); - realTimeTrafficDTO.setSinkSpeed(" Records/S"); - } - realTimeTrafficDTOS.add(realTimeTrafficDTO); - } - jobDetailsVO.setDataNumber(dataNumberDTOS); - jobDetailsVO.setLoadCondition(loadConditionDTOs); - jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); - jobDetailsVO.setLinkisJobInfo(flinkJobInfo); - return Message.ok().data("details", jobDetailsVO); + return Message.ok().data("details", streamTaskService.getJobDetailsVO(streamJob, version)); } @RequestMapping(path = "/execute/history", method = RequestMethod.GET) From f3bee02e8a502e7d4532eb34140320846bf149ce Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 21 Oct 2022 13:41:36 +0800 Subject: [PATCH 017/219] Add config autowired interface to set the log appender params. --- .../collector/StreamisConfigAutowired.java | 15 +++++++++ .../log/collector/StreamisRpcLogAppender.java | 22 +++++++------ .../config/StreamisLogAppenderConfig.java | 33 ++++++++++++------- .../StreamisLogAppenderConfigBuilder.java | 27 --------------- .../src/test/resources/log4j2.xml | 4 +-- .../log/server/storage/JobLogStorage.java | 4 +-- 6 files changed, 53 insertions(+), 52 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java new file mode 100644 index 000000000..25ccfa7d0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; + +/** + * Streamis config autowired + */ +public interface StreamisConfigAutowired { + + /** + * Log appender config + * @param builder builder + */ + StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 737885e1a..1b66ef6ea 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -3,7 +3,6 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfigBuilder; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; import org.apache.logging.log4j.core.Filter; @@ -18,12 +17,10 @@ import org.apache.logging.log4j.core.layout.PatternLayout; import java.io.Serializable; -import java.nio.Buffer; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.ServiceLoader; -import java.util.concurrent.TimeUnit; /** * Streamis rpc log appender @@ -84,16 +81,21 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (Objects.isNull(layout)){ layout = PatternLayout.createDefaultLayout(); } - // Search the config builder - List configBuilders = new ArrayList<>(); + // Search the config autowired class + List configAutowiredEntities = new ArrayList<>(); StreamisLogAppenderConfig logAppenderConfig = null; - ServiceLoader.load(StreamisLogAppenderConfigBuilder.class, - StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configBuilders::add); - if (!configBuilders.isEmpty()){ - logAppenderConfig = configBuilders.get(0).build(applicationName, filter, rpcLogSenderConfig); + ServiceLoader.load(StreamisConfigAutowired.class, + StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configAutowiredEntities::add); + StreamisLogAppenderConfig.Builder builder = new StreamisLogAppenderConfig.Builder(applicationName, filter, rpcLogSenderConfig); + for (StreamisConfigAutowired autowired : configAutowiredEntities){ + logAppenderConfig = autowired.logAppenderConfig(builder); } if (Objects.isNull(logAppenderConfig)){ - logAppenderConfig = new StreamisLogAppenderConfig(applicationName, filter, rpcLogSenderConfig); + logAppenderConfig = builder.build(); + } + applicationName = logAppenderConfig.getApplicationName(); + if (null == applicationName || applicationName.trim().equals("")){ + throw new IllegalArgumentException("Application name cannot be empty"); } return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 40a6a7d0d..14792664e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -23,27 +23,38 @@ public StreamisLogAppenderConfig(String applicationName, Filter filter, this.senderConfig = null != rpcLogSenderConfig? rpcLogSenderConfig : new RpcLogSenderConfig(); } + public static class Builder{ + private String applicationName; + + private Filter filter; + + private RpcLogSenderConfig rpcLogSenderConfig; + + public Builder(String applicationName, Filter filter, + RpcLogSenderConfig rpcLogSenderConfig){ + this.applicationName = applicationName; + this.filter = filter; + this.rpcLogSenderConfig = rpcLogSenderConfig; + } + + StreamisLogAppenderConfig.Builder setAppName(){ + return null; + } + + public StreamisLogAppenderConfig build(){ + return null; + } + } public String getApplicationName() { return applicationName; } - public void setApplicationName(String applicationName) { - this.applicationName = applicationName; - } - public Filter getFilter() { return filter; } - public void setFilter(Filter filter) { - this.filter = filter; - } - public RpcLogSenderConfig getSenderConfig() { return senderConfig; } - public void setSenderConfig(RpcLogSenderConfig senderConfig) { - this.senderConfig = senderConfig; - } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java deleted file mode 100644 index f3374ac48..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; - -import org.apache.logging.log4j.core.Filter; - -import java.util.Map; - -/** - * Config builder for appender - */ -public abstract class StreamisLogAppenderConfigBuilder { - - - /** - * Build method - * @param applicationName application name - * @param filter filter - * @param senderConfig sender config - * @return - */ - public StreamisLogAppenderConfig build(String applicationName, - Filter filter, RpcLogSenderConfig senderConfig){ - return null; - } - - - public abstract Map loadConfigProps(); -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml index d806c5198..56a3bba82 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml @@ -20,8 +20,8 @@ - + diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java index e951a1e7a..66afa1a2b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java @@ -10,9 +10,9 @@ public interface JobLogStorage { /** * Create buckets - * @param jobName job name + * @param appName application name * @param bucketConfig bucket config * @return config */ - JobLogBucket getOrCreateBucket(String jobName, JobLogBucketConfig bucketConfig); + JobLogBucket getOrCreateBucket(String appName, JobLogBucketConfig bucketConfig); } From 62f9c480b4b620c41e015370e9d9c77a82a67de2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Fri, 21 Oct 2022 14:30:15 +0800 Subject: [PATCH 018/219] =?UTF-8?q?fix:=20=E5=8E=86=E5=8F=B2=E6=97=A5?= =?UTF-8?q?=E5=BF=97=EF=BC=8C=E5=85=B3=E9=97=AD=E5=BC=B9=E7=AA=97=E6=97=B6?= =?UTF-8?q?=E9=87=8D=E7=BD=AEendline=E6=95=B0=E6=8D=AE=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E6=97=A5=E5=BF=97=E5=88=86=E9=A1=B5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/logDetail/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/logDetail/index.vue b/web/src/apps/streamis/module/logDetail/index.vue index 17f3a64da..24cce2725 100644 --- a/web/src/apps/streamis/module/logDetail/index.vue +++ b/web/src/apps/streamis/module/logDetail/index.vue @@ -151,6 +151,7 @@ export default { }, cancel() { this.fromLine = 1 + this.endLine = 0 this.spinShow = false this.query = { ignoreKeywords: '', From b31a319cbb98acee22addbc9ce3079e1c2618b6d Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Fri, 21 Oct 2022 14:59:12 +0800 Subject: [PATCH 019/219] fix the compilation bug. --- .../manager/transform/parser/SparkTaskMetricsParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala index 392986b53..ebe8fa9c5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -22,7 +22,7 @@ class SparkTaskMetricsParser extends AbstractTaskMetricsParser { dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit = { - val addDataNumberDTO: String => () = key => { + val addDataNumberDTO: String => Unit = key => { val batch = new JobDetailsVo.DataNumberDTO batch.setDataName(key) batch.setDataNumber(metricsMap.get(key).toString.toInt) From 6ac3092b8bdfdda9cdbd892253634350b7118ee6 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Fri, 21 Oct 2022 20:54:02 +0800 Subject: [PATCH 020/219] fix the long to int parse bug. --- .../manager/transform/parser/SparkTaskMetricsParser.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala index ebe8fa9c5..a8e6bb7c3 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -64,9 +64,9 @@ class SparkTaskMetricsParser extends AbstractTaskMetricsParser { realTimeTrafficDTO.setTransformKey("processing") realTimeTrafficDTO.setSinkKey(metricsMap.getOrDefault("sink", "").asInstanceOf[String]) val sinkSpeed = if (batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) - Utils.msDurationToString(batchMetric.get("totalDelay").asInstanceOf[Long]) + " totalDelay" + Utils.msDurationToString(batchMetric.get("totalDelay").toString.toInt) + " totalDelay" else if (batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) - Utils.msDurationToString(batchMetric.get("taskExecuteTime").asInstanceOf[Long]) + " executeTime(Last Batch)" + Utils.msDurationToString(batchMetric.get("taskExecuteTime").toString.toInt) + " executeTime(Last Batch)" else "" realTimeTrafficDTO.setSinkSpeed(sinkSpeed) case _ => From 58d84103ac14cd94685c8443f41497f5ae480186 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sat, 22 Oct 2022 11:24:26 +0800 Subject: [PATCH 021/219] Complete the log collector (100%) --- .../log/collector/StreamisRpcLogAppender.java | 8 +- .../config/StreamisLogAppenderConfig.java | 203 +++++++++++++++++- .../sender/http/AbstractHttpLogSender.java | 19 +- 3 files changed, 206 insertions(+), 24 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 1b66ef6ea..e086ad17f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -33,17 +33,17 @@ public class StreamisRpcLogAppender extends AbstractAppender { /** * Appender config */ - private StreamisLogAppenderConfig appenderConfig; + private final StreamisLogAppenderConfig appenderConfig; /** * Rpc log sender */ - private StreamisRpcLogSender rpcLogSender; + private final StreamisRpcLogSender rpcLogSender; /** * Cache */ - private LogCache logCache; + private final LogCache logCache; protected StreamisRpcLogAppender(String name, Filter filter, Layout layout, boolean ignoreExceptions, Property[] properties, @@ -53,7 +53,7 @@ protected StreamisRpcLogAppender(String name, Filter filter, this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), this.appenderConfig.getSenderConfig()); this.logCache = this.rpcLogSender.getOrCreateLogCache(); - Runtime.getRuntime().addShutdownHook(new Thread(() -> this.rpcLogSender.close())); + Runtime.getRuntime().addShutdownHook(new Thread(this.rpcLogSender::close)); } @Override diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 14792664e..4f60e0a61 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -1,22 +1,27 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.filter.CompositeFilter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; /** * Appender config */ public class StreamisLogAppenderConfig { - private String applicationName; + private final String applicationName; /** * Filter in log4j */ - private Filter filter; + private final Filter filter; - private RpcLogSenderConfig senderConfig; + private final RpcLogSenderConfig senderConfig; - public StreamisLogAppenderConfig(String applicationName, Filter filter, + StreamisLogAppenderConfig(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; this.filter = filter; @@ -24,26 +29,202 @@ public StreamisLogAppenderConfig(String applicationName, Filter filter, } public static class Builder{ + /** + * Application name + */ private String applicationName; - private Filter filter; + /** + * Filter rules + */ + private final List filters = new ArrayList<>(); - private RpcLogSenderConfig rpcLogSenderConfig; + /** + * Sender config + */ + private final RpcLogSenderConfig rpcLogSenderConfig; public Builder(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - this.filter = filter; - this.rpcLogSenderConfig = rpcLogSenderConfig; + this.filters.add(filter); + this.rpcLogSenderConfig = Optional.ofNullable(rpcLogSenderConfig).orElse(new RpcLogSenderConfig()); + } + + /** + * Set application name + * @param applicationName application name + * @return builder + */ + StreamisLogAppenderConfig.Builder setAppName(String applicationName){ + this.applicationName = applicationName; + return this; + } + + /** + * Set filter + * @param filter filter + * @return builder + */ + StreamisLogAppenderConfig.Builder setFilter(Filter filter){ + this.filters.clear(); + this.filters.add(filter); + return this; + } + + /** + * Append filter + * @param filter filter + * @return builder + */ + StreamisLogAppenderConfig.Builder withFilter(Filter filter){ + filters.add(filter); + return this; + } + + /** + * Rpc connect timeout + * @param connectionTimeout connection timeout + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ + this.rpcLogSenderConfig.setConnectionTimeout(connectionTimeout); + return this; + } + + /** + * Rpc socket timeout + * @param socketTimeout socket timeout + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ + this.rpcLogSenderConfig.setSocketTimeout(socketTimeout); + return this; + } + + /** + * Rpc send retry count + * @param sendRetryCnt send retry count + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ + this.rpcLogSenderConfig.setSendRetryCnt(sendRetryCnt); + return this; + } + + /** + * Rpc server recovery time in seconds + * @param serverRecoveryTimeInSec server recovery time + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ + this.rpcLogSenderConfig.setServerRecoveryTimeInSec(serverRecoveryTimeInSec); + return this; } - StreamisLogAppenderConfig.Builder setAppName(){ - return null; + /** + * Rpc max delay time in seconds + * @param maxDelayTimeInSec max delay time in seconds + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ + this.rpcLogSenderConfig.setMaxDelayTimeInSec(maxDelayTimeInSec); + return this; } + /** + * Rpc auth token code key + * @param tokenCodeKey key of token code + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCodeKey(tokenCodeKey); + return this; + } + + /** + * Rpc auth token user key + * @param tokenUserKey key of token user + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUserKey(tokenUserKey); + return this; + } + + /** + * Rpc auth token user + * @param tokenUser token user + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUser(tokenUser); + return this; + } + + /** + * Rpc auth token code + * @param tokenCode token code + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCode(tokenCode); + return this; + } + + /** + * Rpc cache size + * @param cacheSize cache size + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ + this.rpcLogSenderConfig.getCacheConfig().setSize(cacheSize); + return this; + } + + /** + * Rpc cache max consume thread + * @param maxConsumeThread max consume thread + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThread){ + this.rpcLogSenderConfig.getCacheConfig().setMaxConsumeThread(maxConsumeThread); + return this; + } + + /** + * Rpc buffer size + * @param bufferSize buffer size + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ + this.rpcLogSenderConfig.getBufferConfig().setSize(bufferSize); + return this; + } + + /** + * Rpc buffer expire time in seconds + * @param expireTimeInSec expire time + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireTimeInSec){ + this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); + return this; + } + /** + * Build method + * @return config + */ public StreamisLogAppenderConfig build(){ - return null; + Filter logFilter = null; + if (filters.size() > 1){ + logFilter = CompositeFilter.createFilters(filters.toArray(new Filter[0])); + } else if (!filters.isEmpty()){ + logFilter = filters.get(0); + } + return new StreamisLogAppenderConfig(applicationName, logFilter, rpcLogSenderConfig); } + } public String getApplicationName() { return applicationName; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index d247ba4ad..a0385ecb4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -2,7 +2,6 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcAuthConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; @@ -99,14 +98,16 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th return; } } - - EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); - RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); - postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); - // Ignore the response - postAction.execute(this.globalHttpClient); - // Init the counter - this.exceptionCounter.set(0); + String address = rpcSenderConfig.getAddress(); + if (null != address && !address.trim().equals("")) { + EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); + RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); + postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); + // Ignore the response + postAction.execute(this.globalHttpClient); + // Init the counter + this.exceptionCounter.set(0); + } } } } From 80b5003caea2bab1290bba3614f3959df29cdb3f Mon Sep 17 00:00:00 2001 From: davidhua Date: Sat, 22 Oct 2022 21:12:59 +0800 Subject: [PATCH 022/219] Add Flink log collector and the transform in streamis which resolves the internal configuration --- pom.xml | 1 + .../flink-streamis-log-collector/pom.xml | 67 ++++++++++ .../src/main/assembly/package.xml | 19 +++ .../flink/FlinkStreamisConfigAutowired.java | 104 +++++++++++++++ .../flink/FlinkStreamisConfigDefine.java | 120 ++++++++++++++++++ ....jobmanager.plugin.StreamisConfigAutowired | 1 + .../flink/FlinkConfigurationLoadTest.java | 25 ++++ .../src/test/resources/flink-conf.yaml | 0 .../streamis-job-log-collector/pom.xml | 9 +- .../log/collector/StreamisRpcLogAppender.java | 9 +- .../config/StreamisLogAppenderConfig.java | 42 +++--- .../sender/SendLogCacheConsumer.java | 1 + .../log/collector/sender/http/Json.java | 18 +++ .../StreamisConfigAutowired.java | 4 +- .../collector/StreamisLogAppenderTest.java | 22 +++- .../log/entities/StreamisLogEvents.java | 16 ++- .../jobmanager/manager/conf/JobConf.scala | 17 ++- .../AbstractStreamisTransformJobBuilder.scala | 18 ++- .../impl/FlinkInternalConfigTransform.scala | 42 ++++++ 19 files changed, 501 insertions(+), 34 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/{log/collector => plugin}/StreamisConfigAutowired.java (75%) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala diff --git a/pom.xml b/pom.xml index bd535bf6f..580cce56f 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ 1.1.3 + 4.12 1.1.0 0.2.0 2.11.12 diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml index 5207e035a..6bbee8ba8 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml @@ -16,13 +16,80 @@ 8 1.12.2 + 2.17.1 + 1.7.15 + + com.webank.wedatasphere.streamis + streamis-job-log-collector + ${streamis.version} + org.apache.flink flink-java ${flink.version} + provided + + + org.apache.flink + flink-yarn_2.11 + ${flink.version} + provided + + + + junit + junit + ${junit.version} + test + + + + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + provided + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + provided + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + + + assemble + + single + + + install + + + + + src/main/assembly/package.xml + + false + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml new file mode 100644 index 000000000..8da27bf2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml @@ -0,0 +1,19 @@ + + + package + + + jar + + false + + + / + true + runtime + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java new file mode 100644 index 000000000..a862d080f --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -0,0 +1,104 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.yarn.configuration.YarnConfigOptions; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.filter.LevelMatchFilter; +import org.apache.logging.log4j.core.filter.RegexFilter; + +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; + +import static com.webank.wedatasphere.streamis.jobmanager.log.collector.flink.FlinkStreamisConfigDefine.*; + +/** + * Autoconfigure the streamis config inf Flink environment + */ +public class FlinkStreamisConfigAutowired implements StreamisConfigAutowired { + + /** + * Flink configuration + */ + private Configuration configuration; + + public FlinkStreamisConfigAutowired(){ + // First to load configuration + this.configuration = loadConfiguration(); + } + @Override + public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception{ + String applicationName = this.configuration.getString(YarnConfigOptions.APPLICATION_NAME); + if (StringUtils.isNotBlank(applicationName)){ + builder.setAppName(applicationName); + } + String gateway = this.configuration.getString(LOG_GATEWAY_ADDRESS); + if (StringUtils.isNotBlank(gateway)){ + if (gateway.endsWith("/")){ + gateway = gateway.substring(0, gateway.length() - 1); + } + gateway += this.configuration.getString(LOG_COLLECT_PATH, "/"); + builder.setRpcAddress(gateway); + } + List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); + for(String filterStrategy : filterStrategies){ + if ("LevelMatch".equals(filterStrategy)){ + builder.withFilter(LevelMatchFilter.newBuilder() + .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); + } else if ("RegexMatch".equals(filterStrategy)){ + builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), + null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); + } + } + return builder.setRpcConnTimeout(this.configuration.getInteger(LOG_RPC_CONN_TIMEOUT)) + .setRpcSocketTimeout(this.configuration.getInteger(LOG_RPC_SOCKET_TIMEOUT)) + .setRpcSendRetryCnt(this.configuration.getInteger(LOG_RPC_SEND_RETRY_COUNT)) + .setRpcServerRecoveryTimeInSec(this.configuration.getInteger(LOG_RPC_SERVER_RECOVERY_TIME)) + .setRpcMaxDelayTimeInSec(this.configuration.getInteger(LOG_RPC_MAX_DELAY_TIME)) + .setRpcAuthTokenCodeKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE_KEY)) + .setRpcAuthTokenUserKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER_KEY)) + .setRpcAuthTokenCode(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE)) + .setRpcAuthTokenUser(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER)) + .setRpcCacheSize(this.configuration.getInteger(LOG_RPC_CACHE_SIZE)) + .setRpcCacheMaxConsumeThread(this.configuration.getInteger(LOG_PRC_CACHE_MAX_CONSUME_THREAD)) + .setRpcBufferSize(this.configuration.getInteger(LOG_RPC_BUFFER_SIZE)) + .setRpcBufferExpireTimeInSec(this.configuration.getInteger(LOG_RPC_BUFFER_EXPIRE_TIME)).build(); + } + + /** + * According to : + * String launchCommand = + * BootstrapTools.getTaskManagerShellCommand( + * flinkConfig, + * tmParams, + * ".", + * ApplicationConstants.LOG_DIR_EXPANSION_VAR, + * hasLogback, + * hasLog4j, + * hasKrb5, + * taskManagerMainClass, + * taskManagerDynamicProperties); + * the configuration directory of Flink yarn container is always ".", + * @return configuration + */ + private Configuration loadConfiguration(){ + String configDir = System.getenv("FLINK_CONF_DIR"); + if (null == configDir){ + configDir = "."; + } + Properties properties = System.getProperties(); + Enumeration enumeration = properties.propertyNames(); + Configuration dynamicConfiguration = new Configuration(); + while(enumeration.hasMoreElements()){ + String prop = String.valueOf(enumeration.nextElement()); + dynamicConfiguration.setString(prop, properties.getProperty(prop)); + } + return GlobalConfiguration.loadConfiguration(configDir, dynamicConfiguration); + } + +} diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java new file mode 100644 index 000000000..d40834fa5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java @@ -0,0 +1,120 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import java.util.List; + +/** + * Config definition + */ +public class FlinkStreamisConfigDefine { + + /** + * Gateway address of log module for streamis + */ + public static final ConfigOption LOG_GATEWAY_ADDRESS = ConfigOptions.key("stream.log.gateway.address") + .stringType().noDefaultValue().withDescription("The gateway address ex: http://127.0.0.1:8080"); + + /** + * Entrypoint path of collecting log + */ + public static final ConfigOption LOG_COLLECT_PATH = ConfigOptions.key("stream.log.collect.path") + .stringType().defaultValue("/api/rest_j/v1/streamis/streamJobManager/log/collect/events").withDescription("The entrypoint path of collecting log"); + + /** + * Connection timeout(in milliseconds) in log RPC module + */ + public static final ConfigOption LOG_RPC_CONN_TIMEOUT = ConfigOptions.key("stream.log.rpc.connect-timeout") + .intType().defaultValue(3000).withDescription("Connection timeout(ms) in log RPC module"); + + /** + * Socket timeout(in milliseconds) in log RPC module + */ + public static final ConfigOption LOG_RPC_SOCKET_TIMEOUT = ConfigOptions.key("stream.log.rpc.socket-timeout") + .intType().defaultValue(15000).withDescription("Socket timeout(ms) in log RPC module"); + + /** + * Max retry count of sending message in log RPC module + */ + public static final ConfigOption LOG_RPC_SEND_RETRY_COUNT = ConfigOptions.key("stream.log.rpc.send-retry-count") + .intType().defaultValue(3).withDescription("Max retry count of sending message in log RPC module"); + + /** + * Server recovery time(in seconds) in log RPC module + */ + public static final ConfigOption LOG_RPC_SERVER_RECOVERY_TIME = ConfigOptions.key("stream.log.rpc.server-recovery-time-in-sec") + .intType().defaultValue(5).withDescription("Server recovery time(sec) in log RPC module"); + + /** + * Max delay time(in seconds) in log RPC module. if reach the limit, the message will be dropped + */ + public static final ConfigOption LOG_RPC_MAX_DELAY_TIME = ConfigOptions.key("stream.log.rpc.max-delay-time") + .intType().defaultValue(60).withDescription("Max delay time(sec) in log RPC module"); + + /** + * Token code key in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_CODE_KEY = ConfigOptions.key("stream.log.rpc.auth.token-code-key") + .stringType().defaultValue("Token-Code").withDescription("Token code key in log RPC auth module"); + + /** + * Token user key in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_USER_KEY = ConfigOptions.key("stream.log.rpc.auth.token-user-key") + .stringType().defaultValue("Token-User").withDescription("Token user key in log RPC auth module"); + + /** + * Token code in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_CODE = ConfigOptions.key("stream.log.rpc.auth.token-code") + .stringType().defaultValue("STREAM-LOG").withDescription("Token code in log RPC auth module"); + + /** + * Token user in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_USER = ConfigOptions.key("stream.log.rpc.auth.token-user") + .stringType().defaultValue(System.getProperty("user.name")).withDescription("Token user in log RPC auth module"); + + /** + * Cache size in log RPC module + */ + public static final ConfigOption LOG_RPC_CACHE_SIZE = ConfigOptions.key("stream.log.rpc.cache.size") + .intType().defaultValue(150).withDescription("Cache size in log RPC module"); + + /** + * Max cache consume threads in log RPC module + */ + public static final ConfigOption LOG_PRC_CACHE_MAX_CONSUME_THREAD = ConfigOptions.key("stream.log.rpc.cache.max-consume-thread") + .intType().defaultValue(10).withDescription("Max cache consume threads in log RPC module"); + + /** + * Buffer size in log RPC module + */ + public static final ConfigOption LOG_RPC_BUFFER_SIZE = ConfigOptions.key("stream.log.rpc.buffer.size") + .intType().defaultValue(50).withDescription("Buffer size in log RPC module"); + + /** + * Buffer expire time(sec) in log RPC module + */ + public static final ConfigOption LOG_RPC_BUFFER_EXPIRE_TIME = ConfigOptions.key("stream.log.rpc.buffer.expire-time-in-sec") + .intType().defaultValue(2).withDescription("Buffer expire time (sec) in log RPC module"); + + /** + * Log filter strategy list + */ + public static final ConfigOption> LOG_FILTER_STRATEGIES = ConfigOptions.key("stream.log.filter.strategies") + .stringType().asList().defaultValues("LevelMatch").withDescription("Log filter strategy list"); + + /** + * Level value of LevelMatch filter strategy + */ + public static final ConfigOption LOG_FILTER_LEVEL_MATCH = ConfigOptions.key("stream.log.filter.level-match.level") + .stringType().defaultValue("ERROR").withDescription("Level value of LevelMatch filter strategy"); + + /** + * Regex value of RegexMatch filter strategy + */ + public static final ConfigOption LOG_FILTER_REGEX = ConfigOptions.key("stream.log.filter.regex.value") + .stringType().defaultValue(".*").withDescription("Regex value of RegexMatch filter strategy"); +} diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired new file mode 100644 index 000000000..dc13253b7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired @@ -0,0 +1 @@ +com.webank.wedatasphere.streamis.jobmanager.log.collector.flink.FlinkStreamisConfigAutowired \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java new file mode 100644 index 000000000..d04988099 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.GlobalConfiguration; +import org.junit.Test; + +import java.io.File; +import java.util.Enumeration; +import java.util.Objects; +import java.util.Properties; + +public class FlinkConfigurationLoadTest { + @Test + public void loadConfiguration() { + String configDir = Objects.requireNonNull(FlinkConfigurationLoadTest.class.getResource("/")).getFile(); + Properties properties = System.getProperties(); + Enumeration enumeration = properties.propertyNames(); + Configuration dynamicConfiguration = new Configuration(); + while(enumeration.hasMoreElements()){ + String prop = String.valueOf(enumeration.nextElement()); + dynamicConfiguration.setString(prop, properties.getProperty(prop)); + } + GlobalConfiguration.loadConfiguration(configDir, dynamicConfiguration); + } +} diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index 5fd394795..5c4822539 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -24,7 +24,7 @@ com.webank.wedatasphere.streamis streamis-job-log-common - 0.2.0 + ${streamis.version} @@ -65,5 +65,12 @@ ${log4j.version} provided + + + junit + junit + ${junit.version} + test + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index e086ad17f..5b30c36ce 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -4,7 +4,9 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; @@ -27,9 +29,7 @@ */ @Plugin(name = "StreamRpcLog", category = "Core", elementType = "appender", printObject = true) public class StreamisRpcLogAppender extends AbstractAppender { - private static final String DEFAULT_APPENDER_NAME = "StreamRpcLog"; - /** * Appender config */ @@ -52,6 +52,8 @@ protected StreamisRpcLogAppender(String name, Filter filter, this.appenderConfig = appenderConfig; this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), this.appenderConfig.getSenderConfig()); + this.rpcLogSender.setExceptionListener((subject, t, message) -> + LOGGER.error((null != subject? subject.getClass().getSimpleName() : "") + ": " + message, t)); this.logCache = this.rpcLogSender.getOrCreateLogCache(); Runtime.getRuntime().addShutdownHook(new Thread(this.rpcLogSender::close)); } @@ -74,7 +76,7 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str @PluginAttribute("ignoreExceptions") boolean ignoreExceptions, @PluginElement("Filter") final Filter filter, @PluginElement("Layout") Layout layout, - @PluginElement("RpcLogSender")RpcLogSenderConfig rpcLogSenderConfig){ + @PluginElement("RpcLogSender")RpcLogSenderConfig rpcLogSenderConfig) throws Exception{ if (null == name || name.trim().equals("")){ name = DEFAULT_APPENDER_NAME; } @@ -97,6 +99,7 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (null == applicationName || applicationName.trim().equals("")){ throw new IllegalArgumentException("Application name cannot be empty"); } + LOGGER.info("StreamisRpcLogAppender: init with config {}", Json.toJson(logAppenderConfig, null)); return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 4f60e0a61..0253838ed 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -56,7 +56,7 @@ public Builder(String applicationName, Filter filter, * @param applicationName application name * @return builder */ - StreamisLogAppenderConfig.Builder setAppName(String applicationName){ + public StreamisLogAppenderConfig.Builder setAppName(String applicationName){ this.applicationName = applicationName; return this; } @@ -66,7 +66,7 @@ StreamisLogAppenderConfig.Builder setAppName(String applicationName){ * @param filter filter * @return builder */ - StreamisLogAppenderConfig.Builder setFilter(Filter filter){ + public StreamisLogAppenderConfig.Builder setFilter(Filter filter){ this.filters.clear(); this.filters.add(filter); return this; @@ -77,17 +77,27 @@ StreamisLogAppenderConfig.Builder setFilter(Filter filter){ * @param filter filter * @return builder */ - StreamisLogAppenderConfig.Builder withFilter(Filter filter){ + public StreamisLogAppenderConfig.Builder withFilter(Filter filter){ filters.add(filter); return this; } + /** + * Rpc address + * @param address address + * @return builder + */ + public StreamisLogAppenderConfig.Builder setRpcAddress(String address){ + this.rpcLogSenderConfig.setAddress(address); + return this; + } + /** * Rpc connect timeout * @param connectionTimeout connection timeout * @return builder */ - StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ + public StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ this.rpcLogSenderConfig.setConnectionTimeout(connectionTimeout); return this; } @@ -97,7 +107,7 @@ StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ * @param socketTimeout socket timeout * @return builder */ - StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ + public StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ this.rpcLogSenderConfig.setSocketTimeout(socketTimeout); return this; } @@ -107,7 +117,7 @@ StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ * @param sendRetryCnt send retry count * @return builder */ - StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ + public StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ this.rpcLogSenderConfig.setSendRetryCnt(sendRetryCnt); return this; } @@ -117,7 +127,7 @@ StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ * @param serverRecoveryTimeInSec server recovery time * @return builder */ - StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ + public StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ this.rpcLogSenderConfig.setServerRecoveryTimeInSec(serverRecoveryTimeInSec); return this; } @@ -127,7 +137,7 @@ StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecove * @param maxDelayTimeInSec max delay time in seconds * @return builder */ - StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ + public StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ this.rpcLogSenderConfig.setMaxDelayTimeInSec(maxDelayTimeInSec); return this; } @@ -137,7 +147,7 @@ StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec) * @param tokenCodeKey key of token code * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ this.rpcLogSenderConfig.getAuthConfig().setTokenCodeKey(tokenCodeKey); return this; } @@ -147,7 +157,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ * @param tokenUserKey key of token user * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ this.rpcLogSenderConfig.getAuthConfig().setTokenUserKey(tokenUserKey); return this; } @@ -157,7 +167,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ * @param tokenUser token user * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ this.rpcLogSenderConfig.getAuthConfig().setTokenUser(tokenUser); return this; } @@ -167,7 +177,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ * @param tokenCode token code * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ this.rpcLogSenderConfig.getAuthConfig().setTokenCode(tokenCode); return this; } @@ -177,7 +187,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ * @param cacheSize cache size * @return builder */ - StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ + public StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ this.rpcLogSenderConfig.getCacheConfig().setSize(cacheSize); return this; } @@ -187,7 +197,7 @@ StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ * @param maxConsumeThread max consume thread * @return builder */ - StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThread){ + public StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThread){ this.rpcLogSenderConfig.getCacheConfig().setMaxConsumeThread(maxConsumeThread); return this; } @@ -197,7 +207,7 @@ StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThre * @param bufferSize buffer size * @return builder */ - StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ + public StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ this.rpcLogSenderConfig.getBufferConfig().setSize(bufferSize); return this; } @@ -207,7 +217,7 @@ StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ * @param expireTimeInSec expire time * @return builder */ - StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireTimeInSec){ + public StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireTimeInSec){ this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); return this; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index 52a7b72cb..fac98b90a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -87,6 +87,7 @@ public void run() { if (this.isTerminated && e instanceof InterruptedException){ return; } else { + e.printStackTrace(); System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); // For the unknown exception clear the cache sendBuffer.clear(); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java index 40b20b50e..911ccb381 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java @@ -91,6 +91,24 @@ public static String toJson(Object obj, Class model) { return null; } + public static String toJson(Object obj, Class model, boolean beautify){ + ObjectWriter writer = mapper.writer(); + if(null != obj){ + try{ + if(null != model){ + writer = writer.withView(model); + } + if(beautify){ + return writer.withDefaultPrettyPrinter().writeValueAsString(obj); + } + return writer.writeValueAsString(obj); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + return null; + } + /** * Convert object using serialization and deserialization * diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java similarity index 75% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java index 25ccfa7d0..980a6aec5 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector; +package com.webank.wedatasphere.streamis.jobmanager.plugin; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; @@ -11,5 +11,5 @@ public interface StreamisConfigAutowired { * Log appender config * @param builder builder */ - StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder); + StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java index 189e573ae..d4a7e61ce 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -1,16 +1,28 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector; +import org.apache.logging.log4j.LogManager; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class StreamisLogAppenderTest { private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); - public static void main(String[] args) throws InterruptedException { - while(true){ - for(int i = 0; i < 100; i ++){ - LOG.info("Stream Log appender test"); + @Test + public void appenderLog() throws InterruptedException { + org.apache.logging.log4j.core.Logger logger = (org.apache.logging.log4j.core.Logger) LogManager.getLogger("Test: Hello-world"); + logger.info(""); + int total = 1000; + int tps = 100; + long timer = System.currentTimeMillis() + 1000; + for(int i = 0; i < total; i ++){ + if (i > 0 && i % tps == 0){ + long sleep = timer - System.currentTimeMillis(); + if (sleep > 0){ + Thread.sleep(sleep); + } + timer = System.currentTimeMillis() + 1000; } - Thread.sleep(1000); + LOG.info("Stream Log appender test, sequence id: " + i); } } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 972d1a492..fa0b1c929 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -21,13 +21,19 @@ public StreamisLogEvents(String applicationName, StreamisLogEvent[] events){ this.appName = applicationName; this.events = events; long maxTime = -1; - for(StreamisLogEvent event : events){ - long time = event.getLogTimeStamp(); - if (time > maxTime){ - maxTime = time; + StreamisLogEvent lastEvent = events[events.length - 1]; + if (null == lastEvent) { + for (StreamisLogEvent event : events) { + long time = event.getLogTimeStamp(); + if (time > maxTime) { + maxTime = time; + } } + this.logTimeInMills = maxTime; + }else { + this.logTimeInMills = lastEvent.getLogTimeStamp(); } - this.logTimeInMills = maxTime; + } @Override diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 73697bdbb..33c4827b8 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -15,7 +15,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.conf -import org.apache.linkis.common.conf.{CommonVars, TimeType} +import org.apache.linkis.common.conf.{CommonVars, Configuration, TimeType} import com.webank.wedatasphere.streamis.jobmanager.manager.exception.JobExecuteErrorException @@ -29,6 +29,21 @@ object JobConf { val STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER: CommonVars[String] = CommonVars("wds.streamis.job.param.blank.placeholder", "\u0001") + /** + * Config group for streamis internal configuration + */ + val STREAMIS_INTERNAL_CONFIG_GROUP: CommonVars[String] = CommonVars("wds.streamis.job.internal.config.group", "wds.streamis.internal.params") + + /** + * Gateway for stream job log module + */ + val STREAMIS_JOB_LOG_GATEWAY: CommonVars[String] = CommonVars("wds.streamis.job.log.gateway", Configuration.getGateWayURL()) + + /** + * Path for collecting stream job log + */ + val STREAMIS_JOB_LOG_COLLECT_PATH: CommonVars[String] = CommonVars("wds.streamis.job.log.collect.path", "/api/rest_j/v1/streamis/streamJobManager/log/collect/events") + val FLINK_JOB_STATUS_NOT_STARTED: CommonVars[Int] = CommonVars("wds.streamis.job.status.not-started", 0,"Not Started") val FLINK_JOB_STATUS_COMPLETED: CommonVars[Int] = CommonVars("wds.streamis.job.status.completed", 1,"Completed") diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index 039fbd3e2..6d44dced6 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -18,12 +18,15 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.builder import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.manager.label.entity.engine.RunType.RunType import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.StreamJobMapper import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob import com.webank.wedatasphere.streamis.jobmanager.manager.transform.StreamisTransformJobBuilder import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.{StreamisJobEngineConnImpl, StreamisTransformJob, StreamisTransformJobContent, StreamisTransformJobImpl} import org.springframework.beans.factory.annotation.Autowired +import java.util +import scala.collection.JavaConverters.mapAsJavaMapConverter /** * Created by enjoyyin on 2021/9/22. */ @@ -39,7 +42,12 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB override def build(streamJob: StreamJob): StreamisTransformJob = { val transformJob = createStreamisTransformJob() transformJob.setStreamJob(streamJob) - transformJob.setConfigMap(streamJobConfService.getJobConfig(streamJob.getId)) + val jobConfig: util.Map[String, Any] = Option(streamJobConfService.getJobConfig(streamJob.getId)) + .getOrElse(new util.HashMap[String, Any]()) + // Put and overwrite internal group, users cannot customize the internal configuration + jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, new util.HashMap[String, Any]()) + internalLogConfig(jobConfig) + transformJob.setConfigMap(jobConfig) // transformJob.setConfig(configurationService.getFullTree(streamJob.getId)) val streamJobVersions = streamJobMapper.getJobVersions(streamJob.getId) // 无需判断streamJobVersions是否非空，因为TaskService已经判断了 @@ -48,6 +56,14 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB transformJob } + /** + * Log internal configuration + * @param internal internal config group + */ + private def internalLogConfig(internal: util.Map[String, Any]): Unit = { + internal.put(JobConf.STREAMIS_JOB_LOG_GATEWAY.key, JobConf.STREAMIS_JOB_LOG_GATEWAY.getValue) + internal.put(JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key, JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.getValue) + } } abstract class AbstractFlinkStreamisTransformJobBuilder extends AbstractStreamisTransformJobBuilder{ diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala new file mode 100644 index 000000000..70e9631af --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkInternalConfigTransform.INTERNAL_CONFIG_MAP +import org.apache.linkis.common.conf.CommonVars + +import java.util +import scala.collection.JavaConverters.{mapAsJavaMapConverter, mapAsScalaMapConverter} + +/** + * Flink internal config transform + */ +class FlinkInternalConfigTransform extends FlinkConfigTransform { + + /** + * Config group name + * + * @return + */ + override protected def configGroup(): String = JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue + + override protected def transform(internalConfig: util.Map[String, Any], job: LaunchJob): LaunchJob = { + transformConfig(internalConfig.asScala.map{ + case (key, value) => + (FlinkConfigTransform.FLINK_CONFIG_PREFIX + (INTERNAL_CONFIG_MAP.get(key) match { + case Some(mappingKey) => mappingKey + case _ => value + }), value) + }.asJava, job) + } +} + +object FlinkInternalConfigTransform { + + private val FLINK_LOG_GATEWAY_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-gateway", "stream.log.gateway.address").getValue + + private val FLINK_LOG_COLLECT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-collect-path", "stream.log.collect.path").getValue + + val INTERNAL_CONFIG_MAP = Map(JobConf.STREAMIS_JOB_LOG_GATEWAY.key -> FLINK_LOG_GATEWAY_CONFIG_NAME, + JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key -> FLINK_LOG_COLLECT_PATH_CONFIG_NAME + ) +} From 1c115d526852508897bd4931d9f427a5e80da6e5 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 02:30:01 +0800 Subject: [PATCH 023/219] Complete stream log server to store the log events. --- .../streamis-job-log-server/pom.xml | 2 +- .../log/server/config/StreamJobLogConfig.java | 46 +++ .../log/server/restful/JobLogRestfulApi.java | 48 +++ .../service/DefaultStreamisJobLogService.java | 35 +++ .../server/service/StreamisJobLogService.java | 16 + .../log/server/storage/JobLogStorage.java | 13 +- .../server/storage/StreamisJobLogStorage.java | 127 +++++++- .../server/storage/bucket/JobLogBucket.java | 5 + .../storage/bucket/JobLogBucketConfig.java | 42 +-- .../storage/bucket/JobLogBucketState.java | 14 +- .../storage/bucket/Log4j2JobLogBucket.java | 274 +++++++++++++++++- streamis-server/pom.xml | 6 + .../src/main/resources/linkis.properties | 3 +- 13 files changed, 598 insertions(+), 33 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 7f2a5bc10..ec75cea68 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -9,7 +9,7 @@ 4.0.0 - streamis-job-log-service + streamis-job-log-server 8 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java new file mode 100644 index 000000000..c857d5249 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.config; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +/** + * Store the configuration defined for job log + */ +public class StreamJobLogConfig { + + /** + * Bucket monitor name + */ + public static final CommonVars BUCKET_MONITOR_NAME = CommonVars.apply("wds.stream.job.log.storage.bucket.monitor.name", "Log-Storage-Bucket-Monitor"); + + /** + * Bucket monitor interval + */ + public static final CommonVars BUCKET_MONITOR_INTERVAL = CommonVars.apply("wds.stream.job.log.storage.bucket.monitor.interval", new TimeType("2m")); + + /** + * Bucket max idle time + */ + public static final CommonVars BUCKET_MAX_IDLE_TIME = CommonVars.apply("wds.stream.job.log.storage.bucket.max-idle-time", new TimeType("12h")); + + /** + * Bucket root path + */ + public static final CommonVars BUCKET_ROOT_PATH = CommonVars.apply("wds.stream.job.log.storage.bucket.root-path", "/data/stream/log"); + /** + * Max active part size in bucket + */ + public static final CommonVars BUCKET_MAX_ACTIVE_PART_SIZE = CommonVars.apply("wds.stream.job.log.storage.bucket.max-active-part-size", 100L); + + /** + * Compression of part in bucket + */ + public static final CommonVars BUCKET_PART_COMPRESS = CommonVars.apply("wds.stream.job.log.storage.bucket.part-compress", "gz"); + + /** + * Bucket layout + */ + public static final CommonVars BUCKET_LAYOUT = CommonVars.apply("wds.stream.job.log.storage.bucket.layout", "%msg"); + + public static final CommonVars BUCKET_PART_HOLD_DAY = CommonVars.apply("wds.stream.job.log.storage.bucket.part-hold-day", 30); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java new file mode 100644 index 000000000..7b67c4b2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.restful; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; +import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; +import com.webank.wedatasphere.streamis.jobmanager.log.server.service.StreamisJobLogService; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping(path = "/streamis/streamJobManager/log") +public class JobLogRestfulApi { + + private static final Logger LOG = LoggerFactory.getLogger(JobLogRestfulApi.class); + + @Resource + private StreamisJobLogService streamisJobLogService; + + @RequestMapping(value = "/collect/events", method = RequestMethod.POST) + public Message collectEvents(@RequestBody StreamisLogEvents events, HttpServletRequest request){ + Message result; + try{ + if (StringUtils.isBlank(events.getAppName())){ + return Message.ok("Ignore the stream log events without application name"); + } + String userName = SecurityFilter.getLoginUsername(request); + if (StringUtils.isBlank(userName)){ + throw new StreamJobLogException(-1, "The request should has token user"); + } + this.streamisJobLogService.store(userName, events); + result = Message.ok(); + }catch (Exception e){ + String message = "Fail to collect stream log events, message: " + e.getMessage(); + result = Message.error(message); + } + return result; + } + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java new file mode 100644 index 000000000..8fea4dab6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.service; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.JobLogStorage; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketConfig; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +/** + * Default implement + */ +@Service +public class DefaultStreamisJobLogService implements StreamisJobLogService{ + + @Resource + private JobLogStorage jobLogStorage; + + private JobLogBucketConfig jobLogBucketConfig; + + @PostConstruct + public void init(){ + jobLogBucketConfig = new JobLogBucketConfig(); + } + @Override + public void store(String user, StreamisLogEvents events) { + JobLogBucket jobLogBucket = jobLogStorage.getOrCreateBucket(user, events.getAppName(), jobLogBucketConfig); + // If cannot get log bucket, drop the events + if (null != jobLogBucket){ + jobLogBucket.getBucketStorageWriter().write(events); + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java new file mode 100644 index 000000000..e8f8bfe4e --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.service; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; + +/** + * Job log service + */ +public interface StreamisJobLogService { + + /** + * Store log events + * @param user user own + * @param events events + */ + void store(String user, StreamisLogEvents events); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java index 66afa1a2b..e70590338 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java @@ -10,9 +10,20 @@ public interface JobLogStorage { /** * Create buckets + * @param userName user own * @param appName application name * @param bucketConfig bucket config * @return config */ - JobLogBucket getOrCreateBucket(String appName, JobLogBucketConfig bucketConfig); + JobLogBucket getOrCreateBucket(String userName, String appName, JobLogBucketConfig bucketConfig); + + /** + * Init method + */ + void init(); + + /** + * Destroy method + */ + void destroy(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index b357c30c6..86851e5b7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -1,4 +1,129 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage; -public class StreamisJobLogStorage { +import com.webank.wedatasphere.streamis.jobmanager.log.server.config.StreamJobLogConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketState; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.text.SimpleDateFormat; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static com.webank.wedatasphere.streamis.jobmanager.log.server.config.StreamJobLogConfig.BUCKET_MONITOR_INTERVAL; + +/** + * Job log storage + */ +@Component +public class StreamisJobLogStorage implements JobLogStorage{ + + private static final Logger LOG = LoggerFactory.getLogger(StreamisJobLogStorage.class); + /** + * Buckets + */ + private final Map buckets = new ConcurrentHashMap<>(); + + /** + * Constructor cache + */ + private final Map> bucketConstructors = new ConcurrentHashMap<>(); + /** + * To monitor the status of buckets + */ + private Future monitorThread; + + @Override + public JobLogBucket getOrCreateBucket(String userName, String appName, JobLogBucketConfig bucketConfig) { + String bucketName = toBucketName(userName, appName); + return buckets.computeIfAbsent(bucketName, name -> { + Class bucketClass = bucketConfig.getBucketClass(); + if (Objects.nonNull(bucketClass)) { + Constructor constructor = bucketConstructors.computeIfAbsent(bucketClass.getName(), className -> { + Constructor[] constructors = bucketClass.getConstructors(); + Constructor matchConstructor = null; + for (Constructor constructor1 : constructors) { + Class[] inputParams = constructor1.getParameterTypes(); + if (inputParams.length >= 2 && inputParams[0].equals(String.class) + && inputParams[1].equals(JobLogBucketConfig.class)) { + matchConstructor = constructor1; + break; + } + } + return matchConstructor; + }); + if (Objects.nonNull(constructor)) { + try { + return (JobLogBucket) constructor.newInstance(bucketName, bucketConfig); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + LOG.warn("Cannot create storage log bucket from [{}]", bucketClass.getName(), e); + } + } + } + return null; + }); + } + + @Override + @PostConstruct + public synchronized void init() { + if (Objects.isNull(monitorThread)){ + monitorThread = Utils.defaultScheduler().scheduleAtFixedRate(() -> { + Thread.currentThread().setName(StreamJobLogConfig.BUCKET_MONITOR_NAME.getValue()); + long maxIdleTime = StreamJobLogConfig.BUCKET_MAX_IDLE_TIME.getValue().toLong(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + if (buckets.size() > 0) { + StringBuilder builder = new StringBuilder("Buckets in LogStorage: [\n"); + buckets.forEach((bucketName, bucket) -> { + JobLogBucketState bucketState = bucket.getBucketState(); + builder.append("bucket: [ name: ") + .append(bucketName) + .append(", path: ").append(bucketState.getBucketPath()) + .append(", parts: ").append(bucketState.getBucketParts()) + .append(", write-rate: ").append(bucketState.getBucketWriteRate()).append("/s") + .append(", last-write-time: ").append(dateFormat.format(bucketState.getBucketWriteTime())) + .append(" ]\n"); + if (bucketState.getBucketWriteTime() + maxIdleTime <= System.currentTimeMillis()){ + LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", + bucketName, dateFormat.format(bucketState.getBucketWriteTime())); + bucket.close(); + } + + }); + LOG.info(builder.toString()); + } + + },BUCKET_MONITOR_INTERVAL.getValue().toLong(), BUCKET_MONITOR_INTERVAL.getValue().toLong(), TimeUnit.MILLISECONDS); + } + } + + /** + * Bucket name + * @param userName username + * @param appName app name + * @return bucket name + */ + private String toBucketName(String userName, String appName){ + return userName + "." + appName; + } + + @Override + @PreDestroy + public void destroy() { + // Fist to close all the bucket + buckets.forEach((bucketName, bucket) -> bucket.close()); + if (null != monitorThread){ + monitorThread.cancel(true); + } + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java index c20c08944..b55ce818b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java @@ -17,6 +17,11 @@ public interface JobLogBucket { */ JobLogStorageWriter getBucketStorageWriter(); + /** + * Bucket name + * @return bucket name + */ + String getBucketName(); /** * Close the bucket */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java index 2dc82e40f..831cccc65 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.config.StreamJobLogConfig; import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; import org.apache.linkis.common.conf.CommonVars; @@ -19,7 +20,7 @@ public JobLogBucketConfig(){ this.bucketClass = (Class) defaultBucketClass; } } catch (ClassNotFoundException e) { -// throw new StreamJobLogException.Runtime(-1, "", e); + throw new StreamJobLogException.Runtime(-1, "Cannot find the bucket class, message: " + e.getMessage()); } } @@ -31,7 +32,7 @@ public JobLogBucketConfig(){ /** * Root path for bucket */ - private String bucketRootPath; + private String bucketRootPath = StreamJobLogConfig.BUCKET_ROOT_PATH.getValue(); /** * Attribute @@ -39,29 +40,24 @@ public JobLogBucketConfig(){ protected Map attributes = new HashMap<>(); /** - * Max size of bucket active part + * Max size of bucket active part (MB) */ - private int maxBucketActivePartSize; - - /** - * Max number of bucket part - */ - private int maxBucketPartNum; + private long maxBucketActivePartSize = StreamJobLogConfig.BUCKET_MAX_ACTIVE_PART_SIZE.getValue(); /** * The compress format used for bucket parts */ - private String bucketPartCompress; + private String bucketPartCompress = StreamJobLogConfig.BUCKET_PART_COMPRESS.getValue(); /** - * Max hold time in minutes for bucket part + * Max hold time in days for bucket part */ - private long bucketPartHoldTimeInMin; + private int bucketPartHoldTimeInDay = StreamJobLogConfig.BUCKET_PART_HOLD_DAY.getValue(); /** * Layout pattern */ - private String LogLayOutPattern = "%msg%n"; + private String LogLayOutPattern = StreamJobLogConfig.BUCKET_LAYOUT.getValue(); public Class getBucketClass() { return bucketClass; @@ -87,22 +83,14 @@ public void setAttributes(Map attributes) { this.attributes = attributes; } - public int getMaxBucketActivePartSize() { + public long getMaxBucketActivePartSize() { return maxBucketActivePartSize; } - public void setMaxBucketActivePartSize(int maxBucketActivePartSize) { + public void setMaxBucketActivePartSize(long maxBucketActivePartSize) { this.maxBucketActivePartSize = maxBucketActivePartSize; } - public int getMaxBucketPartNum() { - return maxBucketPartNum; - } - - public void setMaxBucketPartNum(int maxBucketPartNum) { - this.maxBucketPartNum = maxBucketPartNum; - } - public String getBucketPartCompress() { return bucketPartCompress; } @@ -111,12 +99,12 @@ public void setBucketPartCompress(String bucketPartCompress) { this.bucketPartCompress = bucketPartCompress; } - public long getBucketPartHoldTimeInMin() { - return bucketPartHoldTimeInMin; + public int getBucketPartHoldTimeInDay() { + return bucketPartHoldTimeInDay; } - public void setBucketPartHoldTimeInMin(long bucketPartHoldTimeInMin) { - this.bucketPartHoldTimeInMin = bucketPartHoldTimeInMin; + public void setBucketPartHoldTimeInDay(int bucketPartHoldTimeInDay) { + this.bucketPartHoldTimeInDay = bucketPartHoldTimeInDay; } public String getLogLayOutPattern() { diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java index bdd64f265..8051e6d13 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java @@ -11,9 +11,21 @@ public interface JobLogBucketState { */ String getBucketPath(); + /** + * Write rate + * @return rate + */ double getBucketWriteRate(); + /** + * Bucket parts + * @return number + */ int getBucketParts(); - int getBucketWriteTime(); + /** + * Last rite time + * @return time + */ + long getBucketWriteTime(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index 21c3303f5..4f298ecf4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -1,4 +1,276 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; -public class Log4j2JobLogBucket { +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.RollingFileAppender; +import org.apache.logging.log4j.core.appender.rolling.*; +import org.apache.logging.log4j.core.appender.rolling.action.*; +import org.apache.logging.log4j.core.config.AppenderRef; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.checkerframework.checker.units.qual.A; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Job log bucket for log4j + */ +public class Log4j2JobLogBucket implements JobLogBucket{ + + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(Log4j2JobLogBucket.class); + + private static final String DEFAULT_FILE_PATTERN_SUFFIX = ".%d{yyyy-MM-dd}-%i"; + + private static final CommonVars ROLLOVER_MAX = CommonVars.apply("wds.stream.job.log.storage.bucket.log4j.rollover-max", 20); + /** + * Bucket name + */ + private final String bucketName; + + /** + * Logger context + */ + private final LoggerContext loggerContext; + + /** + * Logger entity + */ + private final Logger logger; + + /** + * Storage writer + */ + private final JobLogStorageWriter jobLogStorageWriter; + + /** + * Bucket state + */ + private final JobLogBucketState jobLogBucketState; + + /** + * Last write time; + */ + private long lastWriteTime; + + /** + * Prev Interval time + */ + private long preIntervalTime; + + /** + * Interval counter + */ + private final AtomicLong intervalCounter = new AtomicLong(0); + + /** + * Store the write rate + */ + private double writeRate; + public Log4j2JobLogBucket(String bucketName, JobLogBucketConfig config){ + this.bucketName = bucketName; + // Create logger context + this.loggerContext = (LoggerContext) LogManager.getContext(false); + this.logger = initLogger(this.bucketName, config, this.loggerContext); + this.jobLogStorageWriter = createStorageWriter(); + this.jobLogBucketState = createBucketState(); + } + @Override + public JobLogBucketState getBucketState() { + return this.jobLogBucketState; + } + + @Override + public JobLogStorageWriter getBucketStorageWriter() { + return this.jobLogStorageWriter; + } + + @Override + public String getBucketName() { + return this.bucketName; + } + + @Override + public void close() { + Configuration log4jConfig = this.loggerContext.getConfiguration(); + // First to stop appender + log4jConfig.getAppender(this.bucketName).stop(); + log4jConfig.getLoggerConfig(this.bucketName).removeAppender(this.bucketName); + log4jConfig.removeLogger(this.bucketName); + loggerContext.updateLoggers(); + } + + private synchronized Logger initLogger(String bucketName, JobLogBucketConfig config, LoggerContext loggerContext){ + Configuration log4jConfig = loggerContext.getConfiguration(); + String fileName = resolveFileName(config.getBucketRootPath(), bucketName); + RollingFileAppender appender = RollingFileAppender.newBuilder() + .setLayout(PatternLayout.newBuilder().withPattern(config.getLogLayOutPattern()).build()) + .setName(bucketName) +// .withFileOwner() + .withFileName(fileName) + .withFilePattern(resolveFilePattern(fileName, config.getBucketPartCompress())) + .withStrategy(createRolloverStrategy(log4jConfig, fileName, ROLLOVER_MAX.getValue(), config.getBucketPartHoldTimeInDay())) + .setConfiguration(log4jConfig) + .build(); + appender.start(); + log4jConfig.addAppender(appender); + LoggerConfig loggerConfig = LoggerConfig.newBuilder().withLevel(Level.ALL) + .withRefs(new AppenderRef[]{ + AppenderRef.createAppenderRef(bucketName, null, null) + }) + .withLoggerName(bucketName).withConfig(log4jConfig).build(); + loggerConfig.addAppender(appender, null, null); + log4jConfig.addLogger(bucketName, loggerConfig); + // Should we update the logger context ? + loggerContext.updateLoggers(); + return loggerContext.getLogger(bucketName); + } + + /** + * Create storage writer + * @return storage writer + */ + private JobLogStorageWriter createStorageWriter(){ + return new JobLogStorageWriter() { + @Override + public void write(LogElement logEl) { + String[] contents = logEl.getContents(); + if (null != contents){ + for(String content : contents){ + write(content); + } + } + } + + @Override + public void write(String logLine) { + logger.info(logLine); + long currentTime = System.currentTimeMillis(); + long intervalCnt = intervalCounter.getAndIncrement(); + long intervalTime = (currentTime - preIntervalTime)/1000; + // Per minute accumulate the rate + if ( intervalTime >= 60){ + writeRate = (double)intervalCnt / (double)intervalTime; + preIntervalTime = currentTime; + intervalCounter.set(0); + } + lastWriteTime = currentTime; + + } + + @Override + public void close() { + // Ignore + } + }; + } + + /** + * Create bucket state + * @return bucket state + */ + private JobLogBucketState createBucketState(){ + return new JobLogBucketState() { + private String bucketPath; + @Override + public String getBucketPath() { + if (StringUtils.isBlank(bucketPath)) { + Appender appender = loggerContext.getConfiguration().getAppender(bucketName); + if (appender instanceof RollingFileAppender) { + bucketPath = new File(((RollingFileAppender) appender).getFileName()).getParent(); + } + } + return this.bucketPath; + } + + @Override + public double getBucketWriteRate() { + return writeRate; + } + + @Override + public int getBucketParts() { + AtomicInteger parts = new AtomicInteger(-1); + String bucketPath = getBucketPath(); + if (StringUtils.isNotBlank(bucketPath)){ + Optional.ofNullable(new File(bucketPath).list()).ifPresent(list -> parts.set(list.length)); + } + return parts.get(); + } + + @Override + public long getBucketWriteTime() { + return lastWriteTime; + } + }; + } + /** + * Create rollover strategy + * @param configuration configuration + * @param fileName file name + * @param rolloverMax rollover max inf file pattern + * @param fileHoldDay file hold day time + * @return strategy + */ + private RolloverStrategy createRolloverStrategy(Configuration configuration, + String fileName, int rolloverMax, int fileHoldDay){ + DefaultRolloverStrategy.Builder builder = DefaultRolloverStrategy.newBuilder(); + if (rolloverMax > 0){ + builder.withMax(rolloverMax + ""); + } + if (fileHoldDay > 0){ + // Create the actions to delete old file + builder.withCustomActions(new Action[]{ + DeleteAction.createDeleteAction(new File(fileName).getParent(), false, 2, false, null, + new PathCondition[]{ + IfFileName.createNameCondition(null, ".*"), + IfLastModified.createAgeCondition(Duration.parse(fileHoldDay + "d")) + }, + null, configuration) + } + ); + } + return builder.build(); + } + /** + * Ex: /data/stream/log/hadoop/{projectName}/{jobName}/{projectName}.{jobName}.log + * @param bucketRootPath bucket root path + * @param bucketName bucket name + * @return file name with absolute path + */ + private String resolveFileName(String bucketRootPath, String bucketName){ + // {projectName}.{jobName} + String fileName = FilenameUtils.normalize(bucketName.substring(bucketName.indexOf("."))); + String basePath = bucketRootPath; + if (!basePath.endsWith("/")){ + basePath += "/"; + } + basePath += fileName.replace(".", "/"); + return basePath + "/" + fileName; + } + + /** + * Resolve file pattern + * @param fileName file name + * @param format format + * @return file pattern + */ + private String resolveFilePattern(String fileName, String format){ + String filePattern = fileName + Log4j2JobLogBucket.DEFAULT_FILE_PATTERN_SUFFIX; + if (StringUtils.isNotBlank(format)){ + filePattern = filePattern + (format.startsWith(".") ? format : "." +format); + } + return filePattern; + } } diff --git a/streamis-server/pom.xml b/streamis-server/pom.xml index 701457496..1bf8ecfba 100644 --- a/streamis-server/pom.xml +++ b/streamis-server/pom.xml @@ -69,6 +69,12 @@ ${streamis.version} + + + com.webank.wedatasphere.streamis + streamis-job-log-server + ${streamis.version} + diff --git a/streamis-server/src/main/resources/linkis.properties b/streamis-server/src/main/resources/linkis.properties index b80410c10..b09dbbc4a 100644 --- a/streamis-server/src/main/resources/linkis.properties +++ b/streamis-server/src/main/resources/linkis.properties @@ -34,7 +34,8 @@ wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.streamis.datasou com.webank.wedatasphere.streamis.project.server.restful,\ com.webank.wedatasphere.streamis.jobmanager.restful.api,\ com.webank.wedatasphere.streamis.datasource.execute.rest,\ - com.webank.wedatasphere.streamis.projectmanager.restful.api + com.webank.wedatasphere.streamis.projectmanager.restful.api,\ + com.webank.wedatasphere.streamis.jobmanager.log.server.restful ##mybatis wds.linkis.server.mybatis.mapperLocations=\ classpath*:com/webank/wedatasphere/streamis/datasource/manager/dao/impl/*.xml,\ From 0da93b023a24ab23b875a24855d76e36f89358f8 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 09:29:49 +0800 Subject: [PATCH 024/219] Fix the problem in job builder. --- .../builder/AbstractStreamisTransformJobBuilder.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index 6d44dced6..fedc86591 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -45,8 +45,9 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB val jobConfig: util.Map[String, Any] = Option(streamJobConfService.getJobConfig(streamJob.getId)) .getOrElse(new util.HashMap[String, Any]()) // Put and overwrite internal group, users cannot customize the internal configuration - jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, new util.HashMap[String, Any]()) - internalLogConfig(jobConfig) + val internalGroup = new util.HashMap[String, Any]() + jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, internalGroup) + internalLogConfig(internalGroup) transformJob.setConfigMap(jobConfig) // transformJob.setConfig(configurationService.getFullTree(streamJob.getId)) val streamJobVersions = streamJobMapper.getJobVersions(streamJob.getId) From 7b408281facfafa37315743f588a10404e99f2ea Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 16:31:26 +0800 Subject: [PATCH 025/219] Move the internal group config into JobConfKeyConstants. --- .../jobmanager/launcher/conf/JobConfKeyConstants.scala | 4 ++++ .../streamis/jobmanager/manager/conf/JobConf.scala | 5 ----- .../builder/AbstractStreamisTransformJobBuilder.scala | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala index f756daecc..187b02288 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala @@ -23,6 +23,10 @@ import org.apache.linkis.common.conf.CommonVars */ object JobConfKeyConstants { + /** + * Config group for streamis internal configuration + */ + val GROUP_INTERNAL: CommonVars[String] = CommonVars("wds.streamis.job.internal.config.group", "wds.streamis.internal.params") /** * Group: Flink extra */ diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 33c4827b8..41b91e348 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -29,11 +29,6 @@ object JobConf { val STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER: CommonVars[String] = CommonVars("wds.streamis.job.param.blank.placeholder", "\u0001") - /** - * Config group for streamis internal configuration - */ - val STREAMIS_INTERNAL_CONFIG_GROUP: CommonVars[String] = CommonVars("wds.streamis.job.internal.config.group", "wds.streamis.internal.params") - /** * Gateway for stream job log module */ diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index fedc86591..622961329 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -15,6 +15,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.builder +import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.manager.label.entity.engine.RunType.RunType import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService @@ -46,7 +47,7 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB .getOrElse(new util.HashMap[String, Any]()) // Put and overwrite internal group, users cannot customize the internal configuration val internalGroup = new util.HashMap[String, Any]() - jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, internalGroup) + jobConfig.put(JobConfKeyConstants.GROUP_INTERNAL.getValue, internalGroup) internalLogConfig(internalGroup) transformJob.setConfigMap(jobConfig) // transformJob.setConfig(configurationService.getFullTree(streamJob.getId)) From 7ddb90819c1e7988a63c9347e174185dbdf754ee Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 16:35:52 +0800 Subject: [PATCH 026/219] Move the internal group config into JobConfKeyConstants. --- .../manager/transform/impl/FlinkInternalConfigTransform.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala index 70e9631af..e2d0a7848 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala @@ -1,4 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl +import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkInternalConfigTransform.INTERNAL_CONFIG_MAP @@ -17,7 +18,7 @@ class FlinkInternalConfigTransform extends FlinkConfigTransform { * * @return */ - override protected def configGroup(): String = JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue + override protected def configGroup(): String = JobConfKeyConstants.GROUP_INTERNAL.getValue override protected def transform(internalConfig: util.Map[String, Any], job: LaunchJob): LaunchJob = { transformConfig(internalConfig.asScala.map{ From 503ed3f2dc24a75f5e2eb7b24bb9db2933f41159 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 18:08:51 +0800 Subject: [PATCH 027/219] Make the constraints in transforms be configurable. --- .../impl/FlinkCheckpointConfigTransform.scala | 3 ++- .../impl/FlinkInternalConfigTransform.scala | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala index 01426f656..7d797aaf5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala @@ -23,6 +23,7 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.Checkpoint import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkCheckpointConfigTransform.CHECKPOINT_PATH_CONFIG_NAME +import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.utils.Logging import scala.collection.JavaConverters._ @@ -62,5 +63,5 @@ class FlinkCheckpointConfigTransform extends FlinkConfigTransform with Logging{ } object FlinkCheckpointConfigTransform{ - val CHECKPOINT_PATH_CONFIG_NAME = "state.checkpoints.dir" + private val CHECKPOINT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.checkpoint-path", "state.checkpoints.dir").getValue } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala index e2d0a7848..be9782734 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala @@ -32,12 +32,18 @@ class FlinkInternalConfigTransform extends FlinkConfigTransform { } object FlinkInternalConfigTransform { + /** + * Defined in FlinkStreamisConfigDefine.LOG_GATEWAY_ADDRESS of 'flink-streamis-log-collector' + */ + private val LOG_GATEWAY_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-gateway", "stream.log.gateway.address").getValue - private val FLINK_LOG_GATEWAY_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-gateway", "stream.log.gateway.address").getValue + /** + * Defined in FlinkStreamisConfigDefine.LOG_GATEWAY_ADDRESS of 'flink-streamis-log-collector' + */ + private val LOG_COLLECT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-collect-path", "stream.log.collect.path").getValue - private val FLINK_LOG_COLLECT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-collect-path", "stream.log.collect.path").getValue - val INTERNAL_CONFIG_MAP = Map(JobConf.STREAMIS_JOB_LOG_GATEWAY.key -> FLINK_LOG_GATEWAY_CONFIG_NAME, - JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key -> FLINK_LOG_COLLECT_PATH_CONFIG_NAME + val INTERNAL_CONFIG_MAP = Map(JobConf.STREAMIS_JOB_LOG_GATEWAY.key -> LOG_GATEWAY_CONFIG_NAME, + JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key -> LOG_COLLECT_PATH_CONFIG_NAME ) } From e0d980406b5e3adc160ed0e1a9826b85c447cb01 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 24 Oct 2022 01:26:02 +0800 Subject: [PATCH 028/219] Fix serialization and configuration problem. --- .../flink/FlinkStreamisConfigAutowired.java | 27 +++++++++----- .../log/collector/StreamisRpcLogAppender.java | 4 +- .../config/StreamisLogAppenderConfig.java | 5 ++- .../sender/SendLogExceptionStrategy.java | 1 - .../sender/http/AbstractHttpLogSender.java | 37 +++++++++++++++++-- .../http/request/AbstractHttpAction.java | 1 + .../sender/http/request/EntityPostAction.java | 1 - .../log/entities/StreamisLogEvent.java | 21 +++++++++-- .../log/entities/StreamisLogEvents.java | 22 ++++++++++- .../storage/bucket/Log4j2JobLogBucket.java | 7 ++-- 10 files changed, 101 insertions(+), 25 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index a862d080f..88c38a4b3 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -5,6 +5,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.runtime.util.EnvironmentInformation; import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.Filter; @@ -29,11 +30,13 @@ public class FlinkStreamisConfigAutowired implements StreamisConfigAutowired { public FlinkStreamisConfigAutowired(){ // First to load configuration - this.configuration = loadConfiguration(); + // We should sleep and wait for append of the flink-yaml.conf } @Override public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception{ - String applicationName = this.configuration.getString(YarnConfigOptions.APPLICATION_NAME); + this.configuration = loadConfiguration(); + String applicationName = + this.configuration.getString(YarnConfigOptions.APPLICATION_NAME); if (StringUtils.isNotBlank(applicationName)){ builder.setAppName(applicationName); } @@ -48,13 +51,17 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); for(String filterStrategy : filterStrategies){ if ("LevelMatch".equals(filterStrategy)){ - builder.withFilter(LevelMatchFilter.newBuilder() + builder.withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); } else if ("RegexMatch".equals(filterStrategy)){ builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); } } + String hadoopUser = EnvironmentInformation.getHadoopUser(); + if (hadoopUser.equals("") || hadoopUser.equals("")){ + hadoopUser = ""; + } return builder.setRpcConnTimeout(this.configuration.getInteger(LOG_RPC_CONN_TIMEOUT)) .setRpcSocketTimeout(this.configuration.getInteger(LOG_RPC_SOCKET_TIMEOUT)) .setRpcSendRetryCnt(this.configuration.getInteger(LOG_RPC_SEND_RETRY_COUNT)) @@ -63,7 +70,8 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui .setRpcAuthTokenCodeKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE_KEY)) .setRpcAuthTokenUserKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER_KEY)) .setRpcAuthTokenCode(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE)) - .setRpcAuthTokenUser(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER)) + .setRpcAuthTokenUser(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER, + hadoopUser)) .setRpcCacheSize(this.configuration.getInteger(LOG_RPC_CACHE_SIZE)) .setRpcCacheMaxConsumeThread(this.configuration.getInteger(LOG_PRC_CACHE_MAX_CONSUME_THREAD)) .setRpcBufferSize(this.configuration.getInteger(LOG_RPC_BUFFER_SIZE)) @@ -86,11 +94,12 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui * the configuration directory of Flink yarn container is always ".", * @return configuration */ - private Configuration loadConfiguration(){ - String configDir = System.getenv("FLINK_CONF_DIR"); - if (null == configDir){ - configDir = "."; - } + private synchronized Configuration loadConfiguration(){ +// String configDir = System.getenv("FLINK_CONF_DIR"); +// if (null == configDir){ +// configDir = "."; +// } + String configDir = "."; Properties properties = System.getProperties(); Enumeration enumeration = properties.propertyNames(); Configuration dynamicConfiguration = new Configuration(); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 5b30c36ce..26c304d9b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -99,8 +99,8 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (null == applicationName || applicationName.trim().equals("")){ throw new IllegalArgumentException("Application name cannot be empty"); } - LOGGER.info("StreamisRpcLogAppender: init with config {}", Json.toJson(logAppenderConfig, null)); - return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); + System.out.println("StreamisRpcLogAppender: init with config" + Json.toJson(logAppenderConfig, null)); + return new StreamisRpcLogAppender(name, logAppenderConfig.getFilter(), layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 0253838ed..e334c6be2 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -47,7 +48,9 @@ public static class Builder{ public Builder(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - this.filters.add(filter); + if (Objects.nonNull(filter)) { + this.filters.add(filter); + } this.rpcLogSenderConfig = Optional.ofNullable(rpcLogSenderConfig).orElse(new RpcLogSenderConfig()); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java index c8952ff1d..d33b7d2e4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java @@ -41,7 +41,6 @@ V doSend(Callable sendOperation, SendBuffer sendBuffer){ if (Objects.isNull(retryDescription) || !retryDescription.canRetry) { break; } - } } return null; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index a0385ecb4..827e6666e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -7,13 +7,22 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; +import org.apache.http.client.HttpResponseException; +import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.conn.ConnectTimeoutException; +import org.apache.logging.log4j.core.util.IOUtils; import javax.net.ssl.SSLException; +import javax.sound.midi.SysexMessage; +import javax.xml.ws.Response; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -55,7 +64,6 @@ public int retryCount() { @Override public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { - e.printStackTrace(); boolean shouldRetry = false; // Limit of exception number is the same as the retry times if (exceptionCounter.incrementAndGet() > retryCount()){ @@ -68,6 +76,11 @@ public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBu break; } } + if (!shouldRetry && e instanceof HttpResponseException){ + if (((HttpResponseException) e).getStatusCode() < 500){ + shouldRetry = true; + } + } } if (shouldRetry && !sender.getOrCreateLogCache().isCacheable()){ // Means that the cache is full @@ -103,8 +116,26 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); - // Ignore the response - postAction.execute(this.globalHttpClient); + HttpResponse response = null; + try { + response = postAction.execute(this.globalHttpClient); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode > 200){ + throw new HttpResponseException(statusCode, + IOUtils.toString(new InputStreamReader( + response.getEntity().getContent(), StandardCharsets.UTF_8))); + } + }finally { + // Close the response and release the conn + if (null != response){ + if (response instanceof CloseableHttpResponse){ + ((CloseableHttpResponse)response).close(); + } else { + // Destroy the stream + response.getEntity().getContent().close(); + } + } + } // Init the counter this.exceptionCounter.set(0); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java index 8b0472e08..143f72b8e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java @@ -57,6 +57,7 @@ public HttpResponse execute(HttpClient httpClient) throws IOException { } catch (URISyntaxException e) { throw new IllegalArgumentException("URI maybe has wrong format", e); } + requestHeaders.forEach(requestBase::setHeader); return httpClient.execute(requestBase); } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java index f8bfc5e4c..583483a6f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java @@ -72,7 +72,6 @@ private void addEntityBody(MultipartEntityBuilder builder, String prefix, JsonNo } } else if (node instanceof ValueNode){ ContentType strContent = ContentType.create("text/plain", StandardCharsets.UTF_8); - System.out.println("p: " + prefix + ", data: " + node.asText()); builder.addTextBody(prefix, node.asText(), strContent); } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index 5c82e22cb..3aea21d6f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -12,18 +12,21 @@ public class StreamisLogEvent implements LogElement, Serializable { /** * Log time */ - private final long logTimeInMills; + private long logTimeInMills; /** * Log content */ - private final String content; + private String content; /** * Mark */ private int mark; + public StreamisLogEvent(){ + + } public StreamisLogEvent(String content, long logTimeInMills){ this.content = content; this.logTimeInMills = logTimeInMills; @@ -53,7 +56,19 @@ public int mark() { return this.mark; } - public void mark(int mark){ + public void setLogTimeStamp(long logTimeInMills) { + this.logTimeInMills = logTimeInMills; + } + + public void setContent(String content) { + this.content = content; + } + + public void setMark(int mark) { this.mark = mark; } + + public void setSequenceId(int sequenceId){ + // Ignore + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index fa0b1c929..8e58aa55d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -13,10 +13,12 @@ public class StreamisLogEvents implements LogElement, Serializable { /** * Log time */ - private final long logTimeInMills; + private long logTimeInMills; - private final StreamisLogEvent[] events; + private StreamisLogEvent[] events; + public StreamisLogEvents(){ + } public StreamisLogEvents(String applicationName, StreamisLogEvent[] events){ this.appName = applicationName; this.events = events; @@ -69,4 +71,20 @@ public String getAppName() { public StreamisLogEvent[] getEvents() { return events; } + + public void setAppName(String appName) { + this.appName = appName; + } + + public void setLogTimeStamp(long logTimeInMills) { + this.logTimeInMills = logTimeInMills; + } + + public void setEvents(StreamisLogEvent[] events) { + this.events = events; + } + + public void setSequenceId(int sequenceId){ + // Ignore + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index 4f298ecf4..04cdfd430 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -120,12 +120,13 @@ private synchronized Logger initLogger(String bucketName, JobLogBucketConfig con // .withFileOwner() .withFileName(fileName) .withFilePattern(resolveFilePattern(fileName, config.getBucketPartCompress())) + .withPolicy(SizeBasedTriggeringPolicy.createPolicy(config.getMaxBucketActivePartSize() + "MB")) .withStrategy(createRolloverStrategy(log4jConfig, fileName, ROLLOVER_MAX.getValue(), config.getBucketPartHoldTimeInDay())) .setConfiguration(log4jConfig) .build(); appender.start(); log4jConfig.addAppender(appender); - LoggerConfig loggerConfig = LoggerConfig.newBuilder().withLevel(Level.ALL) + LoggerConfig loggerConfig = LoggerConfig.newBuilder().withAdditivity(false).withLevel(Level.ALL) .withRefs(new AppenderRef[]{ AppenderRef.createAppenderRef(bucketName, null, null) }) @@ -251,13 +252,13 @@ private RolloverStrategy createRolloverStrategy(Configuration configuration, */ private String resolveFileName(String bucketRootPath, String bucketName){ // {projectName}.{jobName} - String fileName = FilenameUtils.normalize(bucketName.substring(bucketName.indexOf("."))); + String fileName = FilenameUtils.normalize(bucketName); String basePath = bucketRootPath; if (!basePath.endsWith("/")){ basePath += "/"; } basePath += fileName.replace(".", "/"); - return basePath + "/" + fileName; + return basePath + "/" + fileName.substring(bucketName.indexOf(".") + 1) + ".log"; } /** From b2052e1dc583582e06abf20aae64b0e8bc9d28be Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Mon, 24 Oct 2022 16:42:44 +0800 Subject: [PATCH 029/219] Optimize the update task and stop task logic. --- .../jobmanager/restful/api/JobRestfulApi.java | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index acb1ac529..7f2e396e3 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -175,19 +175,22 @@ public Message killJob(HttpServletRequest req, StreamJob streamJob = this.streamJobService.getJobById(jobId); if(streamJob == null) { return Message.error("not exists job " + jobId); - } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { - return Message.error("Job " + streamJob.getName() + " is not supported to stop."); } if (!streamJobService.hasPermission(streamJob, userName) && !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to kill/stop StreamJob [" + jobId + "]"); } - try { - PauseResultVo resultVo = streamTaskService.pause(jobId, 0L, userName, Objects.nonNull(snapshot)? snapshot : false); - return snapshot? Message.ok().data("path", resultVo.getSnapshotPath()) : Message.ok(); - } catch (Exception e) { - LOG.error("{} kill job {} failed!", userName, jobId, e); - return Message.error(ExceptionUtils.getRootCauseMessage(e)); + if(JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { + try { + PauseResultVo resultVo = streamTaskService.pause(jobId, 0L, userName, Objects.nonNull(snapshot)? snapshot : false); + return snapshot? Message.ok().data("path", resultVo.getSnapshotPath()) : Message.ok(); + } catch (Exception e) { + LOG.error("{} kill job {} failed!", userName, jobId, e); + return Message.error(ExceptionUtils.getRootCauseMessage(e)); + } + } else { + LOG.error("{} try to kill not-supported-management job {} with name {}.", userName, jobId, streamJob.getName()); + return tryStopTask(req, userName, streamJob, null); } } @@ -334,6 +337,9 @@ public Message updateTask(HttpServletRequest req, if(streamTask == null) { LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); return Message.ok("not exists running task, ignore it."); + } else if(JobConf.isCompleted(streamTask.getStatus())) { + LOG.warn("The task of job {} is completed, ignore to update its metrics.", jobName); + return Message.ok("Task is completed, ignore to update its metrics."); } return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { if(!flinkJobInfo.getApplicationId().equals(appId)) { @@ -367,28 +373,32 @@ public Message stopTask(HttpServletRequest req, } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { return Message.error("Only spark.jar Job support to stop task."); } - if (!streamJobService.hasPermission(streamJobs.get(0), username) && - !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { - return Message.error("Have no permission to stop task for StreamJob [" + jobName + "]."); + return tryStopTask(req, username, streamJobs.get(0), appId); + } + + private Message tryStopTask(HttpServletRequest req, String username, StreamJob streamJob, String appId) { + if (!streamJobService.hasPermission(streamJob, username) && + !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { + return Message.error("Have no permission to stop task for StreamJob [" + streamJob.getName() + "]."); } // 如果存在正在运行的，将其停止掉 - StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { - return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { - if(flinkJobInfo.getApplicationId().equals(appId)) { - LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); + return withFlinkJobInfo(streamJob.getName(), streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if(appId == null || flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", streamJob.getName()); streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); streamTask.setErrDesc("stopped by App itself."); streamTaskService.updateTask(streamTask); return Message.ok(); } else { LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to stop it.", - jobName, flinkJobInfo.getApplicationId(), appId); + streamJob.getName(), flinkJobInfo.getApplicationId(), appId); return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); } }); } else { - LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", jobName); + LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", streamJob.getName()); return Message.ok(); } } From be6ef36e73bb7d6fbff06a306b8caba984eb0bd5 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 24 Oct 2022 17:30:50 +0800 Subject: [PATCH 030/219] Fix the problem in fetching user info when the flink application doesn't have hadoop dependencies. --- .../log/collector/flink/FlinkStreamisConfigAutowired.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index 88c38a4b3..a42be88c1 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -60,7 +60,7 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui } String hadoopUser = EnvironmentInformation.getHadoopUser(); if (hadoopUser.equals("") || hadoopUser.equals("")){ - hadoopUser = ""; + hadoopUser = System.getProperty("user.name"); } return builder.setRpcConnTimeout(this.configuration.getInteger(LOG_RPC_CONN_TIMEOUT)) .setRpcSocketTimeout(this.configuration.getInteger(LOG_RPC_SOCKET_TIMEOUT)) From 7f449ee2b7bc0bfe3cbbc512c01f3c4b7bfb6d9e Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 25 Oct 2022 11:40:06 +0800 Subject: [PATCH 031/219] Add new strategy named ThresholdFilter --- .../log/collector/flink/FlinkStreamisConfigAutowired.java | 6 +++++- .../log/collector/flink/FlinkStreamisConfigDefine.java | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index a42be88c1..c146d23b6 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.filter.LevelMatchFilter; import org.apache.logging.log4j.core.filter.RegexFilter; +import org.apache.logging.log4j.core.filter.ThresholdFilter; import java.util.Enumeration; import java.util.List; @@ -53,7 +54,10 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui if ("LevelMatch".equals(filterStrategy)){ builder.withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); - } else if ("RegexMatch".equals(filterStrategy)){ + }else if ("ThresholdFilter".equals(filterStrategy)){ + builder.withFilter(ThresholdFilter.createFilter(Level + .getLevel(this.configuration.getString(LOG_FILTER_THRESHOLD_MATCH)), Filter.Result.ACCEPT, Filter.Result.DENY)); + }else if ("RegexMatch".equals(filterStrategy)){ builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); } diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java index d40834fa5..379f15e0e 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java @@ -112,6 +112,11 @@ public class FlinkStreamisConfigDefine { public static final ConfigOption LOG_FILTER_LEVEL_MATCH = ConfigOptions.key("stream.log.filter.level-match.level") .stringType().defaultValue("ERROR").withDescription("Level value of LevelMatch filter strategy"); + /** + * Level value of ThresholdMatch filter strategy + */ + public static final ConfigOption LOG_FILTER_THRESHOLD_MATCH = ConfigOptions.key("stream.log.filter.threshold.level") + .stringType().defaultValue("ERROR").withDescription("Level value of ThresholdMatch filter strategy"); /** * Regex value of RegexMatch filter strategy */ From 8b7903ee84259e4637f7385e5b2346de5e7839e0 Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 25 Oct 2022 13:46:01 +0800 Subject: [PATCH 032/219] Fix the problem when closing bucket. --- .../jobmanager/log/server/storage/StreamisJobLogStorage.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index 86851e5b7..abe312b23 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -97,6 +97,8 @@ public synchronized void init() { LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", bucketName, dateFormat.format(bucketState.getBucketWriteTime())); bucket.close(); + // Delete the bucket + buckets.remove(bucketName); } }); From 0ca4ba9128f9bd65ac31cae077bb8b59e72fe000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Tue, 25 Oct 2022 15:49:27 +0800 Subject: [PATCH 033/219] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=9C=A8?= =?UTF-8?q?=E5=88=87=E6=8D=A2=E6=97=A5=E5=BF=97=E7=B1=BB=E5=9E=8B=E6=97=B6?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=97=A5=E5=BF=97=E7=BF=BB=E9=A1=B5=E6=8C=89?= =?UTF-8?q?=E9=92=AE=E5=A4=B1=E6=95=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/logDetail/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/logDetail/index.vue b/web/src/apps/streamis/module/logDetail/index.vue index 24cce2725..ee297746a 100644 --- a/web/src/apps/streamis/module/logDetail/index.vue +++ b/web/src/apps/streamis/module/logDetail/index.vue @@ -177,6 +177,7 @@ export default { }, handleQuery() { this.fromLine = 1 + this.endLine = 0 this.getDatas() } } From 63e3525279b3c24a21ec1283035d261f5f891964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Tue, 25 Oct 2022 17:04:17 +0800 Subject: [PATCH 034/219] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dstreamis?= =?UTF-8?q?=E4=BD=9C=E4=B8=9A=E7=9A=84Flink=E5=8F=82=E6=95=B0=E4=BB=85?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E4=B8=80=E4=B8=AA=E6=97=B6=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E4=BF=9D=E5=AD=98=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobConfig/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/apps/streamis/module/jobConfig/index.vue b/web/src/apps/streamis/module/jobConfig/index.vue index b6bd86f46..860d5af89 100644 --- a/web/src/apps/streamis/module/jobConfig/index.vue +++ b/web/src/apps/streamis/module/jobConfig/index.vue @@ -227,7 +227,7 @@ export default { }); if ((this.diyMap[key] || []).length <= 1) { const only = (this.diyMap[key] || [])[0] || {}; - emptyWarning = !((!only.key || !only.key.trim()) && (!only.value || !only.value.trim())) + emptyWarning = (!only.key || !only.key.trim()) && (!only.value || !only.value.trim()) } }); console.log('configuration', configuration, this.valueMap) From 1c90f871fad5c60bab730cbd0ae68cdfbe4b819d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Tue, 25 Oct 2022 18:44:51 +0800 Subject: [PATCH 035/219] =?UTF-8?q?fix:=20=E6=97=A0=E6=9D=83=E9=99=90?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=81=9C=E6=AD=A2=E5=B7=A5=E4=BD=9C=E6=B5=81?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E6=97=B6=E5=9C=A8=E5=BC=B9=E7=AA=97=E4=B8=AD?= =?UTF-8?q?=E4=BD=93=E7=8E=B0=E5=A4=B1=E8=B4=A5=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobList/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index c02f0ea0a..64551087a 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -742,6 +742,7 @@ export default { this.queryProcess(bulk_sbj); } catch (error) { console.warn(error); + this.modalContent = '停止任务失败，失败信息：' + error // this.modalLoading = false } }, From cc67dbb56472583bdb26f17c871579ce6bdb6e47 Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 26 Oct 2022 05:10:46 +0800 Subject: [PATCH 036/219] Split the log collector into log4j1 and log4j2; Avoid the jackson module in log collector. --- pom.xml | 2 +- .../flink-streamis-log-collector/pom.xml | 0 .../src/main/assembly/package.xml | 0 .../flink/FlinkStreamisConfigAutowired.java | 25 ++- .../flink/FlinkStreamisConfigDefine.java | 0 ....jobmanager.plugin.StreamisConfigAutowired | 0 .../flink/FlinkConfigurationLoadTest.java | 1 - .../src/test/resources/flink-conf.yaml | 0 .../streamis-job-log-collector-core/pom.xml | 39 ++++ .../log/collector/ExceptionListener.java | 0 .../log/collector/cache/LogCache.java | 0 .../log/collector/config/RpcAuthConfig.java | 22 +- .../collector/config/RpcLogSenderConfig.java | 38 ++-- .../collector/config/SendBufferConfig.java | 24 +-- .../collector/config/SendLogCacheConfig.java | 25 +-- .../config/StreamisLogAppenderConfig.java | 67 +------ .../sender/AbstractRpcLogSender.java | 0 .../log/collector/sender/RpcLogSender.java | 0 .../log/collector/sender/SendLogCache.java | 0 .../sender/SendLogCacheConsumer.java | 0 .../sender/SendLogExceptionStrategy.java | 0 .../sender/StreamisRpcLogSender.java | 5 + .../sender/buf/AbstractSendBuffer.java | 0 .../sender/buf/ImmutableSendBuffer.java | 0 .../log/collector/sender/buf/SendBuffer.java | 0 .../sender/http/AbstractHttpLogSender.java | 41 +++- .../collector/sender/http/HttpClientTool.java | 0 .../sender/http/entities/Resource.java | 0 .../http/request/AbstractHttpAction.java | 0 .../sender/http/request/GetAction.java | 0 .../sender/http/request/HttpAction.java | 0 .../sender/http/request/StringPostAction.java | 29 +++ .../plugin/StreamisConfigAutowired.java | 0 .../streamis-job-log-collector/pom.xml | 15 +- .../log/collector/StreamisRpcLogAppender.java | 10 +- .../log4j2/StreamisLog4j2AppenderConfig.java | 88 ++++++++ .../log4j2/config/RpcAuthConfig.java | 31 +++ .../log4j2/config/RpcLogSenderConfig.java | 37 ++++ .../log4j2/config/SendBufferConfig.java | 28 +++ .../log4j2/config/SendLogCacheConfig.java | 27 +++ .../collector/StreamisLogAppenderTest.java | 3 - .../src/test/resources/log4j2.xml | 4 +- .../streamis-job-log-collector1x/pom.xml | 55 +++++ .../log/collector/StreamisRpcLogAppender.java | 189 ++++++++++++++++++ .../log4j1/StreamisLog4jAppenderConfig.java | 95 +++++++++ .../collector/StreamisLogAppenderTest.java | 27 +++ .../src/test/resources/log4j.properties | 43 ++++ streamis-jobmanager/streamis-job-log/pom.xml | 6 +- .../log/collector/sender/http/Json.java | 149 -------------- .../sender/http/request/EntityPostAction.java | 79 -------- .../sender/http/request/PostAction.java | 41 ---- .../streamis-job-log-common/pom.xml | 12 +- .../jobmanager/log/entities/LogElement.java | 1 + .../log/entities/StreamisLogEvent.java | 14 +- .../log/entities/StreamisLogEvents.java | 25 ++- .../jobmanager/log/json/JsonTool.java | 45 +++++ .../server/entities/StreamisLogEvents.java | 12 ++ .../log/server/restful/JobLogRestfulApi.java | 2 +- .../storage/bucket/Log4j2JobLogBucket.java | 1 + .../jobmanager/log/LogStorageTest.java | 4 + 60 files changed, 907 insertions(+), 454 deletions(-) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/pom.xml (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/assembly/package.xml (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java (79%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java (97%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/test/resources/flink-conf.yaml (100%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java (68%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java (72%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java (54%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java (55%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java (77%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java (91%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java (84%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java (100%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/pom.xml (77%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java (93%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java (80%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/src/test/resources/log4j2.xml (91%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java diff --git a/pom.xml b/pom.xml index 580cce56f..0fc4d2fba 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ 1.8 3.3.3 2.8.5 - 2.11.3 + 2.13.2 3.1.1 4.5.4 4.5.4 diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/assembly/package.xml similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/assembly/package.xml diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java similarity index 79% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index c146d23b6..ba1756006 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.StreamisLog4j2AppenderConfig; import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; import org.apache.commons.lang3.StringUtils; import org.apache.flink.configuration.Configuration; @@ -49,17 +50,19 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui gateway += this.configuration.getString(LOG_COLLECT_PATH, "/"); builder.setRpcAddress(gateway); } - List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); - for(String filterStrategy : filterStrategies){ - if ("LevelMatch".equals(filterStrategy)){ - builder.withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) - .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); - }else if ("ThresholdFilter".equals(filterStrategy)){ - builder.withFilter(ThresholdFilter.createFilter(Level - .getLevel(this.configuration.getString(LOG_FILTER_THRESHOLD_MATCH)), Filter.Result.ACCEPT, Filter.Result.DENY)); - }else if ("RegexMatch".equals(filterStrategy)){ - builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), - null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); + if (builder instanceof StreamisLog4j2AppenderConfig.Builder) { + List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); + for (String filterStrategy : filterStrategies) { + if ("LevelMatch".equals(filterStrategy)) { + ((StreamisLog4j2AppenderConfig.Builder)builder).withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) + .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); + } else if ("ThresholdFilter".equals(filterStrategy)) { + ((StreamisLog4j2AppenderConfig.Builder)builder).withFilter(ThresholdFilter.createFilter(Level + .getLevel(this.configuration.getString(LOG_FILTER_THRESHOLD_MATCH)), Filter.Result.ACCEPT, Filter.Result.DENY)); + } else if ("RegexMatch".equals(filterStrategy)) { + ((StreamisLog4j2AppenderConfig.Builder)builder).withFilter(RegexFilter.createFilter(this.configuration.getString(LOG_FILTER_REGEX), + null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); + } } } String hadoopUser = EnvironmentInformation.getHadoopUser(); diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java similarity index 97% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java index d04988099..6958957a7 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java @@ -4,7 +4,6 @@ import org.apache.flink.configuration.GlobalConfiguration; import org.junit.Test; -import java.io.File; import java.util.Enumeration; import java.util.Objects; import java.util.Properties; diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/resources/flink-conf.yaml similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/resources/flink-conf.yaml diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml new file mode 100644 index 000000000..bc68d0796 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -0,0 +1,39 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-collector-core + + + 8 + 8 + 4.5.13 + 4.5.4 + + + + + com.webank.wedatasphere.streamis + streamis-job-log-common + ${streamis.version} + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpmine.version} + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java similarity index 68% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java index e88eb91e9..ebf9b7f68 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java @@ -1,17 +1,9 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; /** * Authentication config */ -@Plugin( - name = "AuthConfig", - category = "Core", - printObject = true -) public class RpcAuthConfig { /** * Key of token-code @@ -52,12 +44,6 @@ public RpcAuthConfig(String tokenCodeKey, String tokenCode, String tokenUserKey, } } - @PluginFactory - public static RpcAuthConfig createRpcAuthConfig(@PluginAttribute("tokenCodeKey") String tokenCodeKey, - @PluginAttribute("tokenCode") String tokenCode, - @PluginAttribute("tokenUserKey") String tokenUserKey, @PluginAttribute("tokenUser") String tokenUser){ - return new RpcAuthConfig(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); - } public String getTokenCodeKey() { return tokenCodeKey; } @@ -89,4 +75,12 @@ public String getTokenCode() { public void setTokenCode(String tokenCode) { this.tokenCode = tokenCode; } + + @Override + public String toString() { + return "RpcAuthConfig{" + + ", tokenUserKey='" + tokenUserKey + '\'' + + ", tokenUser='" + tokenUser + '\'' + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java similarity index 72% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java index 10a063946..0fbc563d7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java @@ -1,21 +1,11 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginElement; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.util.Integers; import java.util.Objects; /** * Rpc sender configuration */ -@Plugin( - name = "RpcLogSender", - category = "Core", - printObject = true -) public class RpcLogSenderConfig { /** @@ -86,18 +76,7 @@ public RpcLogSenderConfig(String address, int sendRetryCnt, int connectionTimeou this.bufferConfig = bufferConfig; } } - @PluginFactory - public static RpcLogSenderConfig createConfig( - @PluginAttribute("address") String address, @PluginAttribute("sendRetryCnt") String sendRetryCnt, - @PluginAttribute("connectionTimeout") String connectionTimeout, @PluginAttribute("socketTimeout") String socketTimeout, - @PluginAttribute("serverRecoveryTimeInSec") String serverRecoveryTimeInSec, @PluginAttribute("maxDelayTimeInSec") String maxDelayTimeInSec, - @PluginElement("AuthConfig")RpcAuthConfig authConfig, @PluginElement("SendLogCache") SendLogCacheConfig cacheConfig, - @PluginElement("SendBuffer")SendBufferConfig bufferConfig){ - return new RpcLogSenderConfig(address, Integers.parseInt(sendRetryCnt, 3), - Integers.parseInt(connectionTimeout, 3000), Integers.parseInt(socketTimeout, 15000), - Integers.parseInt(serverRecoveryTimeInSec, 5), Integers.parseInt(maxDelayTimeInSec, 60), - authConfig, cacheConfig, bufferConfig); - } + public RpcAuthConfig getAuthConfig() { return authConfig; } @@ -169,4 +148,19 @@ public int getServerRecoveryTimeInSec() { public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; } + + @Override + public String toString() { + return "RpcLogSenderConfig{" + + "address='" + address + '\'' + + ", connectionTimeout=" + connectionTimeout + + ", socketTimeout=" + socketTimeout + + ", sendRetryCnt=" + sendRetryCnt + + ", serverRecoveryTimeInSec=" + serverRecoveryTimeInSec + + ", maxDelayTimeInSec=" + maxDelayTimeInSec + + ", authConfig=" + authConfig + + ", cacheConfig=" + cacheConfig + + ", bufferConfig=" + bufferConfig + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java similarity index 54% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java index 651caa616..6be0ae826 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java @@ -1,15 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.util.Integers; - -@Plugin( - name = "SendBuffer", - category = "Core", - printObject = true -) + public class SendBufferConfig { /** * Size of send buffer @@ -30,11 +21,6 @@ public SendBufferConfig(int size, long expireTimeInSec){ this.expireTimeInSec = expireTimeInSec; } - @PluginFactory - public static SendBufferConfig createBufferConfig( - @PluginAttribute("size") String size, @PluginAttribute("expireTimeInSec") String expireTimeInSec){ - return new SendBufferConfig(Integers.parseInt(size, 50), Integers.parseInt(expireTimeInSec, 2)); - } public int getSize() { return size; } @@ -50,4 +36,12 @@ public long getExpireTimeInSec() { public void setExpireTimeInSec(long expireTimeInSec) { this.expireTimeInSec = expireTimeInSec; } + + @Override + public String toString() { + return "SendBufferConfig{" + + "size=" + size + + ", expireTimeInSec=" + expireTimeInSec + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java similarity index 55% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java index 6921584a5..1caaedb21 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java @@ -1,18 +1,9 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.util.Integers; - /** * Cache config */ -@Plugin( - name = "SendLogCache", - category = "Core", - printObject = true -) + public class SendLogCacheConfig { /** * Size of send cache @@ -32,11 +23,7 @@ public SendLogCacheConfig(int size, int maxConsumeThread){ this.size = size; this.maxConsumeThread = maxConsumeThread; } - @PluginFactory - public static SendLogCacheConfig createCacheConfig( - @PluginAttribute("size") String size, @PluginAttribute("maxConsumeThread") String maxConsumeThread){ - return new SendLogCacheConfig(Integers.parseInt(size, 150), Integers.parseInt(maxConsumeThread, 10)); - } + public int getSize() { return size; } @@ -52,4 +39,12 @@ public int getMaxConsumeThread() { public void setMaxConsumeThread(int maxConsumeThread) { this.maxConsumeThread = maxConsumeThread; } + + @Override + public String toString() { + return "SendLogCacheConfig{" + + "size=" + size + + ", maxConsumeThread=" + maxConsumeThread + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java similarity index 77% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index e334c6be2..35680913f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -1,10 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.Filter; -import org.apache.logging.log4j.core.filter.CompositeFilter; - import java.util.ArrayList; -import java.util.List; import java.util.Objects; import java.util.Optional; @@ -13,19 +9,13 @@ */ public class StreamisLogAppenderConfig { - private final String applicationName; + protected final String applicationName; - /** - * Filter in log4j - */ - private final Filter filter; - private final RpcLogSenderConfig senderConfig; + protected final RpcLogSenderConfig senderConfig; - StreamisLogAppenderConfig(String applicationName, Filter filter, - RpcLogSenderConfig rpcLogSenderConfig){ + protected StreamisLogAppenderConfig(String applicationName, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - this.filter = filter; this.senderConfig = null != rpcLogSenderConfig? rpcLogSenderConfig : new RpcLogSenderConfig(); } @@ -33,24 +23,17 @@ public static class Builder{ /** * Application name */ - private String applicationName; - - /** - * Filter rules - */ - private final List filters = new ArrayList<>(); + protected String applicationName; /** * Sender config */ - private final RpcLogSenderConfig rpcLogSenderConfig; + protected final RpcLogSenderConfig rpcLogSenderConfig; - public Builder(String applicationName, Filter filter, + public Builder(String applicationName, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - if (Objects.nonNull(filter)) { - this.filters.add(filter); - } + this.rpcLogSenderConfig = Optional.ofNullable(rpcLogSenderConfig).orElse(new RpcLogSenderConfig()); } @@ -64,26 +47,7 @@ public StreamisLogAppenderConfig.Builder setAppName(String applicationName){ return this; } - /** - * Set filter - * @param filter filter - * @return builder - */ - public StreamisLogAppenderConfig.Builder setFilter(Filter filter){ - this.filters.clear(); - this.filters.add(filter); - return this; - } - /** - * Append filter - * @param filter filter - * @return builder - */ - public StreamisLogAppenderConfig.Builder withFilter(Filter filter){ - filters.add(filter); - return this; - } /** * Rpc address @@ -224,28 +188,15 @@ public StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireT this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); return this; } - /** - * Build method - * @return config - */ + public StreamisLogAppenderConfig build(){ - Filter logFilter = null; - if (filters.size() > 1){ - logFilter = CompositeFilter.createFilters(filters.toArray(new Filter[0])); - } else if (!filters.isEmpty()){ - logFilter = filters.get(0); - } - return new StreamisLogAppenderConfig(applicationName, logFilter, rpcLogSenderConfig); + return new StreamisLogAppenderConfig(applicationName, rpcLogSenderConfig); } - } public String getApplicationName() { return applicationName; } - public Filter getFilter() { - return filter; - } public RpcLogSenderConfig getSenderConfig() { return senderConfig; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java similarity index 91% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java index b9ec15537..d019c29f5 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -37,4 +37,9 @@ protected StreamisLogEvents aggregateBuffer(SendBuffer sendBuf } return null; } + + @Override + protected String convertToJsonString(StreamisLogEvents aggregatedEntity) { + return aggregatedEntity.toJson(); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java similarity index 84% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index 827e6666e..c693d0152 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -5,23 +5,18 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.StringPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.HttpResponseException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.conn.ConnectTimeoutException; -import org.apache.logging.log4j.core.util.IOUtils; import javax.net.ssl.SSLException; -import javax.sound.midi.SysexMessage; -import javax.xml.ws.Response; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.InterruptedIOException; +import java.io.*; import java.net.UnknownHostException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -113,7 +108,7 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th } String address = rpcSenderConfig.getAddress(); if (null != address && !address.trim().equals("")) { - EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); + StringPostAction postAction = new StringPostAction(rpcSenderConfig.getAddress(), convertToJsonString(aggregatedEntity)); RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); HttpResponse response = null; @@ -122,8 +117,7 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th int statusCode = response.getStatusLine().getStatusCode(); if (statusCode > 200){ throw new HttpResponseException(statusCode, - IOUtils.toString(new InputStreamReader( - response.getEntity().getContent(), StandardCharsets.UTF_8))); + convertToString(response.getEntity().getContent(), StandardCharsets.UTF_8)); } }finally { // Close the response and release the conn @@ -141,4 +135,29 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th } } } + + /** + * Convert input to string + * @param inputStream input stream + * @param charset charset + * @return string value + * @throws IOException + */ + private String convertToString(InputStream inputStream, Charset charset) throws IOException { + StringBuilder builder = new StringBuilder(); + try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, charset))){ + String line; + while((line = reader.readLine()) != null){ + builder.append(line); + } + } + return builder.toString(); + } + + /** + * Convert the entity to json + * @param aggregatedEntity aggregated entity + * @return json string + */ + protected abstract String convertToJsonString(E aggregatedEntity); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java new file mode 100644 index 000000000..6ce0d8cdf --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; + +/** + * Use string to request + */ +public class StringPostAction extends AbstractHttpAction { + + /** + * Raw string value + */ + private final String rawString; + public StringPostAction(String uri, String rawString) { + super(uri); + this.rawString = rawString; + } + + @Override + protected HttpPost getRequestMethod() { + HttpPost httpPost = new HttpPost(); + StringEntity stringEntity = new StringEntity(rawString, "UTF-8"); + stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); + httpPost.setEntity(stringEntity); + return httpPost; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml similarity index 77% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index 5c4822539..e2e556fc8 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -14,8 +14,6 @@ 8 8 - 4.5.13 - 4.5.4 2.17.1 1.7.15 @@ -23,20 +21,9 @@ com.webank.wedatasphere.streamis - streamis-job-log-common + streamis-job-log-collector-core ${streamis.version} - - - org.apache.httpcomponents - httpclient - ${httpclient.version} - - - org.apache.httpcomponents - httpmime - ${httpmine.version} - org.slf4j diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java similarity index 93% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 26c304d9b..c4674106f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -3,8 +3,8 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.StreamisLog4j2AppenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; import org.apache.logging.log4j.core.Filter; @@ -85,12 +85,12 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str } // Search the config autowired class List configAutowiredEntities = new ArrayList<>(); - StreamisLogAppenderConfig logAppenderConfig = null; + StreamisLog4j2AppenderConfig logAppenderConfig = null; ServiceLoader.load(StreamisConfigAutowired.class, StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configAutowiredEntities::add); - StreamisLogAppenderConfig.Builder builder = new StreamisLogAppenderConfig.Builder(applicationName, filter, rpcLogSenderConfig); + StreamisLog4j2AppenderConfig.Builder builder = new StreamisLog4j2AppenderConfig.Builder(applicationName, filter, rpcLogSenderConfig); for (StreamisConfigAutowired autowired : configAutowiredEntities){ - logAppenderConfig = autowired.logAppenderConfig(builder); + logAppenderConfig = (StreamisLog4j2AppenderConfig) autowired.logAppenderConfig(builder); } if (Objects.isNull(logAppenderConfig)){ logAppenderConfig = builder.build(); @@ -99,7 +99,7 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (null == applicationName || applicationName.trim().equals("")){ throw new IllegalArgumentException("Application name cannot be empty"); } - System.out.println("StreamisRpcLogAppender: init with config" + Json.toJson(logAppenderConfig, null)); + System.out.println("StreamisRpcLogAppender: init with config => " + logAppenderConfig); return new StreamisRpcLogAppender(name, logAppenderConfig.getFilter(), layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java new file mode 100644 index 000000000..f68d77210 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java @@ -0,0 +1,88 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.filter.CompositeFilter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Appender config for log4j2 + */ +public class StreamisLog4j2AppenderConfig extends StreamisLogAppenderConfig { + /** + * Filter in log4j2 + */ + private final Filter filter; + + public StreamisLog4j2AppenderConfig(String applicationName, Filter filter, + RpcLogSenderConfig rpcLogSenderConfig){ + super(applicationName, rpcLogSenderConfig); + this.filter = filter; + } + + public static class Builder extends StreamisLogAppenderConfig.Builder { + + /** + * Filter rules + */ + private final List filters = new ArrayList<>(); + + public Builder(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig) { + super(applicationName, rpcLogSenderConfig); + if (Objects.nonNull(filter)) { + this.filters.add(filter); + } + } + + /** + * Set filter + * @param filter filter + * @return builder + */ + public StreamisLog4j2AppenderConfig.Builder setFilter(Filter filter){ + this.filters.clear(); + this.filters.add(filter); + return this; + } + + /** + * Append filter + * @param filter filter + * @return builder + */ + public StreamisLog4j2AppenderConfig.Builder withFilter(Filter filter){ + filters.add(filter); + return this; + } + + /** + * Build method + * @return config + */ + public StreamisLog4j2AppenderConfig build(){ + Filter logFilter = null; + if (filters.size() > 1){ + logFilter = CompositeFilter.createFilters(filters.toArray(new Filter[0])); + } else if (!filters.isEmpty()){ + logFilter = filters.get(0); + } + return new StreamisLog4j2AppenderConfig(applicationName, logFilter, rpcLogSenderConfig); + } + } + public Filter getFilter() { + return filter; + } + + @Override + public String toString() { + return "StreamisLog4j2AppenderConfig{" + + "applicationName='" + applicationName + '\'' + + ", senderConfig=" + senderConfig + + ", filter=" + filter + + '}'; + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java new file mode 100644 index 000000000..87a10ba85 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; + +/** + * AuthConfig Element in log4j2 + */ +@Plugin( + name = "AuthConfig", + category = "Core", + printObject = true +) +public class RpcAuthConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcAuthConfig { + + public RpcAuthConfig(){ + super(); + } + public RpcAuthConfig(String tokenCodeKey, String tokenCode, String tokenUserKey, String tokenUser) { + super(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); + } + + @PluginFactory + public static RpcAuthConfig createRpcAuthConfig(@PluginAttribute("tokenCodeKey") String tokenCodeKey, + @PluginAttribute("tokenCode") String tokenCode, + @PluginAttribute("tokenUserKey") String tokenUserKey, @PluginAttribute("tokenUser") String tokenUser){ + return new RpcAuthConfig(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); + } + +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java new file mode 100644 index 000000000..7be526465 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +/** + * Rpc sender configuration + */ +@Plugin( + name = "RpcLogSender", + category = "Core", + printObject = true +) +public class RpcLogSenderConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig { + + public RpcLogSenderConfig(String address, int sendRetryCnt, int connectionTimeout, int socketTimeout, int serverRecoveryTimeInSec, int maxDelayTimeInSec, + RpcAuthConfig authConfig, SendLogCacheConfig cacheConfig, SendBufferConfig bufferConfig) { + super(address, sendRetryCnt, connectionTimeout, socketTimeout, serverRecoveryTimeInSec, maxDelayTimeInSec, authConfig, cacheConfig, bufferConfig); + } + + @PluginFactory + public static RpcLogSenderConfig createConfig( + @PluginAttribute("address") String address, @PluginAttribute("sendRetryCnt") String sendRetryCnt, + @PluginAttribute("connectionTimeout") String connectionTimeout, @PluginAttribute("socketTimeout") String socketTimeout, + @PluginAttribute("serverRecoveryTimeInSec") String serverRecoveryTimeInSec, @PluginAttribute("maxDelayTimeInSec") String maxDelayTimeInSec, + @PluginElement("AuthConfig")RpcAuthConfig authConfig, @PluginElement("SendLogCache") SendLogCacheConfig cacheConfig, + @PluginElement("SendBuffer")SendBufferConfig bufferConfig){ + return new RpcLogSenderConfig(address, Integers.parseInt(sendRetryCnt, 3), + Integers.parseInt(connectionTimeout, 3000), Integers.parseInt(socketTimeout, 15000), + Integers.parseInt(serverRecoveryTimeInSec, 5), Integers.parseInt(maxDelayTimeInSec, 60), + authConfig, cacheConfig, bufferConfig); + } + +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java new file mode 100644 index 000000000..936accd72 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +@Plugin( + name = "SendBuffer", + category = "Core", + printObject = true +) +public class SendBufferConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.SendBufferConfig { + + public SendBufferConfig() { + } + + public SendBufferConfig(int size, long expireTimeInSec) { + super(size, expireTimeInSec); + } + + @PluginFactory + public static SendBufferConfig createBufferConfig( + @PluginAttribute("size") String size, @PluginAttribute("expireTimeInSec") String expireTimeInSec){ + return new SendBufferConfig(Integers.parseInt(size, 50), + Integers.parseInt(expireTimeInSec, 2)); + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java new file mode 100644 index 000000000..f4a63c49c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +/** + * Cache config + */ +@Plugin( + name = "SendLogCache", + category = "Core", + printObject = true +) +public class SendLogCacheConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.SendLogCacheConfig { + + public SendLogCacheConfig(int size, int maxConsumeThread) { + super(size, maxConsumeThread); + } + + @PluginFactory + public static SendLogCacheConfig createCacheConfig( + @PluginAttribute("size") String size, @PluginAttribute("maxConsumeThread") String maxConsumeThread){ + return new SendLogCacheConfig(Integers.parseInt(size, 150), Integers.parseInt(maxConsumeThread, 10)); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java similarity index 80% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java index d4a7e61ce..2cec0ec46 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector; -import org.apache.logging.log4j.LogManager; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,8 +8,6 @@ public class StreamisLogAppenderTest { private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); @Test public void appenderLog() throws InterruptedException { - org.apache.logging.log4j.core.Logger logger = (org.apache.logging.log4j.core.Logger) LogManager.getLogger("Test: Hello-world"); - logger.info(""); int total = 1000; int tps = 100; long timer = System.currentTimeMillis() + 1000; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/resources/log4j2.xml similarity index 91% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/resources/log4j2.xml index 56a3bba82..ee3f4125a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/resources/log4j2.xml @@ -21,9 +21,9 @@ + socketTimeout="20000" serverRecoveryTimeInSec="5" maxDelayTimeInSec="60"> - + ` diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml new file mode 100644 index 000000000..43792639c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -0,0 +1,55 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-collector1x + + + 8 + 8 + 1.2.17 + 1.7.12 + + + + com.webank.wedatasphere.streamis + streamis-job-log-collector-core + ${streamis.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + provided + + + + log4j + log4j + ${log4j.version} + provided + + + + junit + junit + ${junit.version} + test + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java new file mode 100644 index 000000000..ccbaf02fc --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -0,0 +1,189 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j1.StreamisLog4jAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.SimpleLayout; +import org.apache.log4j.helpers.LogLog; +import org.apache.log4j.spi.LoggingEvent; + +import java.util.*; + +/** + * Rpc appender for log4j1 + */ +public class StreamisRpcLogAppender extends AppenderSkeleton { + + /** + * Application name + */ + private String applicationName; + + /** + * Appender config + */ + private StreamisLog4jAppenderConfig appenderConfig; + + /** + * Rpc log sender + */ + private StreamisRpcLogSender rpcLogSender; + + /** + * Rpc log sender config + */ + private RpcLogSenderConfig rpcLogSenderConfig = new RpcLogSenderConfig(); + + + /** + * Cache + */ + private LogCache logCache; + + + @Override + protected void append(LoggingEvent loggingEvent) { + String content = super.getLayout().format(loggingEvent); + // Transform to stream log event; + StreamisLogEvent logEvent = new StreamisLogEvent(content, System.currentTimeMillis()); + if (Objects.nonNull(logCache)){ + try { + this.logCache.cacheLog(logEvent); + } catch (InterruptedException e) { + LogLog.error("StreamisRpcLogAppender: " + this.getName() + + " interrupted when cache the log into the RPC sender, message: " + e.getMessage()); + } + } + } + + @Override + public void close() { + if (Objects.nonNull(this.rpcLogSender)){ + this.rpcLogSender.close(); + } + } + + @Override + public boolean requiresLayout() { + return true; + } + + @Override + public void activateOptions() { + if (Objects.nonNull(this.logCache)){ + return; + } + if (Objects.isNull(getLayout())){ + setLayout(new SimpleLayout()); + } + // Search the config autowired class + List configAutowiredEntities = new ArrayList<>(); + StreamisLog4jAppenderConfig logAppenderConfig = null; + ServiceLoader.load(StreamisConfigAutowired.class, + StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configAutowiredEntities::add); + StreamisLog4jAppenderConfig.Builder builder = new StreamisLog4jAppenderConfig.Builder(this.applicationName, + getThreshold(), getFilter(), rpcLogSenderConfig); + for (StreamisConfigAutowired autowired : configAutowiredEntities){ + try { + logAppenderConfig = (StreamisLog4jAppenderConfig) autowired.logAppenderConfig(builder); + } catch (Exception e) { + LogLog.warn("Unable to autowired the config from: " +autowired.getClass().getName(), e); + } + } + if (Objects.isNull(logAppenderConfig)){ + logAppenderConfig = builder.build(); + } + this.applicationName = logAppenderConfig.getApplicationName(); + if (null == applicationName || applicationName.trim().equals("")){ + throw new IllegalArgumentException("Application name cannot be empty"); + } + this.appenderConfig = logAppenderConfig; + // Set the threshold to error default + setThreshold(Optional.ofNullable(logAppenderConfig.getThreshold()).orElse(Level.ERROR)); + // First to clear the filters + clearFilters(); + // Then to add filter + logAppenderConfig.getFilters().forEach(this :: addFilter); + System.out.println("StreamisRpcLogAppender: init with config => " + logAppenderConfig); + this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), + this.appenderConfig.getSenderConfig()); + this.rpcLogSender.setExceptionListener((subject, t, message) -> + LogLog.error((null != subject? subject.getClass().getSimpleName() : "") + ": " + message, t)); + this.logCache = this.rpcLogSender.getOrCreateLogCache(); + } + + + public String getAppName() { + return applicationName; + } + + /** + * Application name + * @param applicationName name + */ + public void setAppName(String applicationName) { + this.applicationName = applicationName; + } + + public void setRpcAddress(String address){ + this.rpcLogSenderConfig.setAddress(address); + } + + public void setRpcConnTimeout(int connectionTimeout){ + this.rpcLogSenderConfig.setConnectionTimeout(connectionTimeout); + } + + public void setRpcSocketTimeout(int socketTimeout){ + this.rpcLogSenderConfig.setSocketTimeout(socketTimeout); + } + public void setRpcSendRetryCnt(int sendRetryCnt){ + this.rpcLogSenderConfig.setSendRetryCnt(sendRetryCnt); + } + + public void setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ + this.rpcLogSenderConfig.setServerRecoveryTimeInSec(serverRecoveryTimeInSec); + } + + public void setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ + this.rpcLogSenderConfig.setMaxDelayTimeInSec(maxDelayTimeInSec); + } + // Authentication + public void setRpcAuthTokenCodeKey(String tokenCodeKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCodeKey(tokenCodeKey); + } + + public void setRpcAuthTokenUserKey(String tokenUserKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUserKey(tokenUserKey); + } + + public void setRpcAuthTokenUser(String tokenUser){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUser(tokenUser); + } + + public void setRpcAuthTokenCode(String tokenCode){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCode(tokenCode); + } + + // Cache configuration + public void setRpcCacheSize(int cacheSize){ + this.rpcLogSenderConfig.getCacheConfig().setSize(cacheSize); + } + + public void setRpcCacheMaxConsumeThread(int maxConsumeThread){ + this.rpcLogSenderConfig.getCacheConfig().setMaxConsumeThread(maxConsumeThread); + } + + // Buffer configuration + public void setRpcBufferSize(int bufferSize){ + this.rpcLogSenderConfig.getBufferConfig().setSize(bufferSize); + } + + public void setRpcBufferExpireTimeInSec(int expireTimeInSec){ + this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java new file mode 100644 index 000000000..84489c155 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java @@ -0,0 +1,95 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j1; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import org.apache.log4j.Priority; +import org.apache.log4j.spi.Filter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Appender config for log4j1 + */ +public class StreamisLog4jAppenderConfig extends StreamisLogAppenderConfig { + + /** + * Filter in log4j1 + */ + private final List filters = new ArrayList<>(); + /** + * + */ + private final Priority threshold; + + protected StreamisLog4jAppenderConfig(String applicationName, Priority threshold, List filters, + RpcLogSenderConfig rpcLogSenderConfig) { + super(applicationName, rpcLogSenderConfig); + this.threshold = threshold; + this.filters.addAll(filters); + } + + public static class Builder extends StreamisLogAppenderConfig.Builder{ + + /** + * Filter rules + */ + private final List filters = new ArrayList<>(); + + /** + * Threshold + */ + private Priority threshold; + + public Builder(String applicationName, Priority threshold, Filter filter,RpcLogSenderConfig rpcLogSenderConfig) { + super(applicationName, rpcLogSenderConfig); + this.threshold = threshold; + if (Objects.nonNull(filter)) { + this.filters.add(filter); + } + } + + public StreamisLog4jAppenderConfig.Builder setFilter(Filter filter){ + this.filters.clear(); + this.filters.add(filter); + return this; + } + + public StreamisLog4jAppenderConfig.Builder withFilter(Filter filter){ + filters.add(filter); + return this; + } + + /** + * Set threshold + * @param threshold threshold + * @return builder + */ + public StreamisLog4jAppenderConfig.Builder threshold(Priority threshold){ + this.threshold = threshold; + return this; + } + public StreamisLog4jAppenderConfig build(){ + return new StreamisLog4jAppenderConfig(applicationName, threshold, filters, rpcLogSenderConfig); + } + } + + public List getFilters() { + return filters; + } + + public Priority getThreshold() { + return threshold; + } + + @Override + public String toString() { + return "StreamisLog4jAppenderConfig{" + + "applicationName='" + applicationName + '\'' + + ", senderConfig=" + senderConfig + + ", filters=" + filters + + ", threshold=" + threshold + + '}'; + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java new file mode 100644 index 000000000..0dcca02c9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import org.apache.log4j.PropertyConfigurator; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamisLogAppenderTest { + private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); + @Test + public void appenderLog() throws InterruptedException { + PropertyConfigurator.configure(StreamisLogAppenderTest.class.getResource("/log4j.properties").getPath()); + int total = 1000; + int tps = 100; + long timer = System.currentTimeMillis() + 1000; + for(int i = 0; i < total; i ++){ + if (i > 0 && i % tps == 0){ + long sleep = timer - System.currentTimeMillis(); + if (sleep > 0){ + Thread.sleep(sleep); + } + timer = System.currentTimeMillis() + 1000; + } + LOG.info("Stream Log appender test, sequence id: " + i); + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties new file mode 100644 index 000000000..85e2a9167 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties @@ -0,0 +1,43 @@ +# +# Copyright 2021 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,stream + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + +log4j.appender.stream=com.webank.wedatasphere.streamis.jobmanager.log.collector.StreamisRpcLogAppender +log4j.appender.stream.appName=stream_applicatioin +log4j.appender.stream.Threshold=INFO +log4j.appender.stream.layout=org.apache.log4j.PatternLayout +log4j.appender.stream.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.stream.rpcConnTimeout=3000 +log4j.appender.stream.rpcSocketTimeout=15000 +log4j.appender.stream.rpcSendRetryCnt=3 +log4j.appender.stream.rpcServerRecoveryTimeInSec=5 +log4j.appender.stream.rpcMaxDelayTimeInSec=60 +log4j.appender.stream.rpcAuthTokenCodeKey= +log4j.appender.stream.rpcAuthTokenUserKey= +log4j.appender.stream.rpcAuthTokenUser= +log4j.appender.stream.rpcAuthTokenCode= +log4j.appender.stream.rpcCacheSize=200 +log4j.appender.stream.rpcCacheMaxConsumeThread=1 +log4j.appender.stream.rpcBufferSize=50 +log4j.appender.stream.rpcBufferExpireTimeInSec=2 \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml index 3446a972b..0c8b087f1 100644 --- a/streamis-jobmanager/streamis-job-log/pom.xml +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -12,8 +12,10 @@ streamis-job-log pom - streamis-job-log-collector - flink-streamis-log-collector + job-log-collector/streamis-job-log-collector-core + job-log-collector/streamis-job-log-collector + job-log-collector/streamis-job-log-collector1x + job-log-collector/flink-streamis-log-collector streamis-job-log-server streamis-job-log-common diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java deleted file mode 100644 index 911ccb381..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java +++ /dev/null @@ -1,149 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.*; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; - -/** - * Json utils - */ -public class Json { - private static final String PREFIX = "["; - private static final String SUFFIX = "]"; - - private static ObjectMapper mapper; - - static{ - mapper = new ObjectMapper(); - mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); - mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); - mapper.configure(DeserializationFeature.READ_ENUMS_USING_TO_STRING, true); - mapper.configure(SerializationFeature.WRITE_ENUMS_USING_TO_STRING, true); - mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); - //empty beans allowed - mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - //cancel to scape non ascii - mapper.configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, false); - } - private Json(){} - - @SuppressWarnings("unchecked") - public static T fromJson(String json, Class clazz, Class... parameters) throws Exception{ - if(null != json && !json.trim().equals("")){ - try{ - if(parameters.length > 0){ - return (T)mapper.readValue(json, mapper.getTypeFactory().constructParametricType(clazz, parameters)); - } - if(json.startsWith(PREFIX) - && json.endsWith(SUFFIX)){ - JavaType javaType = mapper.getTypeFactory() - .constructParametricType(ArrayList.class, clazz); - return mapper.readValue(json, javaType); - } - return (T)mapper.readValue(json, clazz); - } catch (Exception e) { - String message = "Unable to deserialize to object from string(json) in type: [" + - (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; - throw new RuntimeException(message, e); - } - } - return null; - } - - public static T fromJson(InputStream stream, Class clazz, Class... parameters) throws Exception{ - StringBuilder builder = new StringBuilder(); - String jsonStr = null; - try{ - BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); - while((jsonStr = reader.readLine()) != null){ - builder.append(jsonStr); - } - reader.close(); - }catch(Exception e){ - String message = "Unable to deserialize to object from stream(json) in type: [" + - (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; - throw new RuntimeException(message, e); - } - return fromJson(builder.toString(), clazz, parameters); - } - - public static String toJson(Object obj, Class model) { - ObjectWriter writer = mapper.writer(); - if(null != obj){ - try{ - if(null != model){ - writer = writer.withView(model); - } - return writer.writeValueAsString(obj); - } catch (JsonProcessingException e) { - String message = "Unable to serialize the object in type: ["+ (null != model ? model.getSimpleName() : "UNKNOWN") + "]"; - throw new RuntimeException(message, e); - } - } - return null; - } - - public static String toJson(Object obj, Class model, boolean beautify){ - ObjectWriter writer = mapper.writer(); - if(null != obj){ - try{ - if(null != model){ - writer = writer.withView(model); - } - if(beautify){ - return writer.withDefaultPrettyPrinter().writeValueAsString(obj); - } - return writer.writeValueAsString(obj); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - return null; - } - - /** - * Convert object using serialization and deserialization - * - * @param simpleObj simpleObj - * @param tClass type class - * @param T - * @return result - */ - @SuppressWarnings("unchecked") - public static T convert(Object simpleObj, Class tClass, Class... parameters) { - try { - if (parameters.length > 0) { - return mapper.convertValue(simpleObj, mapper.getTypeFactory().constructParametricType(tClass, parameters)); - } - return (T) mapper.convertValue(simpleObj, tClass); - } catch (Exception e) { - String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + - ", " + tClass.getSimpleName() + ": "+ Class.class + ", ...: " + Class.class + ")"; - throw new RuntimeException(message, e); - } - } - - public static T convert(Object simpleObj, JavaType javaType) { - try { - return mapper.convertValue(simpleObj, javaType); - } catch (Exception e) { - String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + - ", " + javaType.getTypeName() + ": "+ JavaType.class + ")"; - throw new RuntimeException(message, e); - } - } - - public static ObjectMapper getMapper(){ - return mapper; - } - -} - diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java deleted file mode 100644 index 583483a6f..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.ValueNode; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.entities.Resource; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import java.nio.charset.StandardCharsets; -import java.util.*; - -/** - * Post action for entity - */ -public class EntityPostAction extends PostAction{ - - private static final String RESOURCE_FIELD_NAME = "resources"; - - private T entity; - - public EntityPostAction(String uri, T entity) { - super(uri); - if (Objects.isNull(entity)){ - throw new IllegalArgumentException("Entity cannot be null in action"); - } - this.entity = entity; - } - - @Override - HttpEntity getHttpEntity() { - try { - if (entity instanceof Resource) { - MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - String entityStr = Json.toJson(entity, null); - JsonNode rootNode = Json.getMapper().readTree(entityStr); - addEntityBody(entityBuilder, "", rootNode); - Optional.ofNullable(((Resource) entity).getResources()).ifPresent(resources -> { - for(int i = 0; i < resources.size(); i ++){ - entityBuilder.addBinaryBody(RESOURCE_FIELD_NAME + "[" + i +"]", resources.get(i)); - } - }); - return entityBuilder.build(); - } else { - StringEntity stringEntity; - stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(entity, null)), "UTF-8"); - stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); - return stringEntity; - } - } catch (Exception e) { - throw new IllegalArgumentException("Cannot deserialize the entity: " + entity.getClass().getName(), e); - } - } - - private void addEntityBody(MultipartEntityBuilder builder, String prefix, JsonNode node){ - if (node instanceof ObjectNode){ - if (null != prefix && !prefix.trim().equals("")){ - prefix += "."; - } - Iterator> fields = node.fields(); - while (fields.hasNext()){ - Map.Entry entry = fields.next(); - addEntityBody(builder, prefix + entry.getKey(), entry.getValue()); - } - } else if (node instanceof ArrayNode){ - ArrayNode arrayNode = ((ArrayNode)node); - for (int i = 0 ; i < arrayNode.size(); i ++){ - addEntityBody(builder, prefix + "[" + i + "]", arrayNode.get(i)); - } - } else if (node instanceof ValueNode){ - ContentType strContent = ContentType.create("text/plain", StandardCharsets.UTF_8); - builder.addTextBody(prefix, node.asText(), strContent); - } - } - -} \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java deleted file mode 100644 index 9b3ea003f..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; - -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; - -import java.util.Objects; - -/** - * Post action - */ -public class PostAction extends AbstractHttpAction { - - public PostAction(String uri) { - super(uri); - } - - /** - * Http entity (use stringEntity and "application/json" default) - * @return http entity - */ - HttpEntity getHttpEntity(){ - StringEntity stringEntity; - try{ - stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(getRequestPayload(), null)), "UTF-8"); - } catch (Exception e){ - throw new IllegalArgumentException("Cannot deserialize the requestPayLoad: " + getRequestPayload().toString(), e); - } - stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); - return stringEntity; - } - - @Override - protected HttpPost getRequestMethod() { - HttpPost httpPost = new HttpPost(); - httpPost.setEntity(getHttpEntity()); - return httpPost; - } -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index a675aa404..4a171b938 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -14,15 +14,15 @@ 8 8 - 2.13.2.2 + - - com.fasterxml.jackson.core - jackson-databind - ${jackson-databind.version} - + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index e253d2a32..da3a7054b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -30,4 +30,5 @@ public interface LogElement { * @return */ int mark(); + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index 3aea21d6f..fe340e066 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -1,8 +1,10 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -import com.fasterxml.jackson.annotation.JsonIgnore; + +import com.webank.wedatasphere.streamis.jobmanager.log.json.JsonTool; import java.io.Serializable; +import java.util.Objects; /** * Log event for streamis @@ -42,7 +44,6 @@ public long getLogTimeStamp() { } @Override - @JsonIgnore public String[] getContents() { return new String[]{content}; } @@ -71,4 +72,13 @@ public void setMark(int mark) { public void setSequenceId(int sequenceId){ // Ignore } + + public String toJson(){ + return "{" + + "\"logTimeStamp\":" + logTimeInMills + + ",\"content\":" + (Objects.isNull(content)? null : "\"" + JsonTool.encodeStrValue(content) + "\"") + + ",\"sequenceId\":0" + + "}"; + + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 8e58aa55d..7e942b675 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -1,8 +1,9 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -import com.fasterxml.jackson.annotation.JsonIgnore; +import com.webank.wedatasphere.streamis.jobmanager.log.json.JsonTool; import java.io.Serializable; +import java.util.Objects; public class StreamisLogEvents implements LogElement, Serializable { @@ -50,7 +51,6 @@ public long getLogTimeStamp() { @Override - @JsonIgnore public String[] getContents() { String[] contents = new String[events.length]; for(int i = 0 ; i < contents.length; i++){ @@ -87,4 +87,25 @@ public void setEvents(StreamisLogEvent[] events) { public void setSequenceId(int sequenceId){ // Ignore } + + public String toJson(){ + return "{" + + "\"logTimeStamp\":" + logTimeInMills + + ",\"appName\":" + (Objects.isNull(appName)? null : "\"" + JsonTool.encodeStrValue(appName) + "\"") + + ",\"events\":[" + + (Objects.isNull(events) || events.length <=0 ? "" : joinEvents(events, ",") ) + "]" + + ",\"sequenceId\":0" + + "}"; + } + + private String joinEvents(StreamisLogEvent[] events, String separator){ + StringBuilder builder = new StringBuilder(); + for(int i = 0; i < events.length; i ++){ + builder.append(events[i].toJson()); + if (i < events.length - 1){ + builder.append(separator); + } + } + return builder.toString(); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java new file mode 100644 index 000000000..d59749261 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.json; + +public class JsonTool { + + /** + * Avoid the special char + * @param input input string + * @return output string + */ + public static String encodeStrValue(String input){ + char[] chars = input.toCharArray(); + StringBuilder sb = new StringBuilder(); + for (char c : chars) { + switch (c) { + case '\"': + sb.append("\\\""); + break; + case '\\': + sb.append("\\\\"); + break; + case '/': + sb.append("\\/"); + break; + case '\b': + sb.append("\\b"); + break; + case '\f': + sb.append("\\f"); + break; + case '\n': + sb.append("\\n"); + break; + case '\r': + sb.append("\\r"); + break; + case '\t': + sb.append("\\t"); + break; + default: + sb.append(c); + } + } + return sb.toString(); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java new file mode 100644 index 000000000..8676c5778 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.entities; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class StreamisLogEvents extends com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents { + + @Override + @JsonIgnore + public String[] getContents() { + return super.getContents(); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java index 7b67c4b2c..595a63e50 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.restful; -import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; +import com.webank.wedatasphere.streamis.jobmanager.log.server.entities.StreamisLogEvents; import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; import com.webank.wedatasphere.streamis.jobmanager.log.server.service.StreamisJobLogService; import org.apache.commons.lang.StringUtils; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index 04cdfd430..0915e0fa0 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -274,4 +274,5 @@ private String resolveFilePattern(String fileName, String format){ } return filePattern; } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java new file mode 100644 index 000000000..6ced214f2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log; + +public class LogStorageTest { +} From 157526dc21336a0cd2c181d8663a7014668edb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Wed, 26 Oct 2022 11:06:07 +0800 Subject: [PATCH 037/219] =?UTF-8?q?fix:=20=E5=85=B3=E9=97=AD=E5=BC=B9?= =?UTF-8?q?=E7=AA=97=E6=97=B6=E6=B8=85=E7=A9=BA=E5=BF=AB=E7=85=A7=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E6=95=B0=E6=8D=AE=EF=BC=8C=E9=81=BF=E5=85=8D=E5=AF=B9?= =?UTF-8?q?=E4=B8=8B=E6=AC=A1=E5=BC=B9=E7=AA=97=E4=BF=A1=E6=81=AF=E9=80=A0?= =?UTF-8?q?=E6=88=90=E5=B9=B2=E6=89=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobList/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 64551087a..23ea4bd5d 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -783,6 +783,7 @@ export default { onClose(status) { if (status) return; this.processModalVisable = false; + this.snapPaths = [] this.isFinish = false; clearTimeout(this.timer); this.failTasks = []; From 84cdb539f832fb0361f993545d74d33303c7fd69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Wed, 26 Oct 2022 14:29:19 +0800 Subject: [PATCH 038/219] =?UTF-8?q?fix:=20=E4=B8=8A=E4=BC=A0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=A1=A8=E5=8D=95=E6=A0=A1=E9=AA=8C=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/i18n/en.json | 4 ++- web/src/apps/streamis/i18n/zh.json | 4 ++- .../apps/streamis/module/uploadFile/index.vue | 31 +++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/web/src/apps/streamis/i18n/en.json b/web/src/apps/streamis/i18n/en.json index 8bf799ba2..fad63b15f 100644 --- a/web/src/apps/streamis/i18n/en.json +++ b/web/src/apps/streamis/i18n/en.json @@ -179,7 +179,8 @@ "selectJar": "Please choose Jar", "upload": "Upload", "choosedJar" : "Choosed Jar", - "jarError": "File must be Jar" + "jarError": "File must be Jar", + "choosedFile": "Choosed File" }, "projectFile": { "file": "File", @@ -187,6 +188,7 @@ "overrideImport": "Override import", "chooseUploadFile": "Please choose the import resource file", "versionPlaceholder": "Only number and ., such as 0.1.0", + "commentLength": "The length of comment text is too long", "versionEmpty": "Version Cannot be empty", "fileEmpty": "File cannot be empty", "delete": "Delete", diff --git a/web/src/apps/streamis/i18n/zh.json b/web/src/apps/streamis/i18n/zh.json index d19676419..13eccbe64 100644 --- a/web/src/apps/streamis/i18n/zh.json +++ b/web/src/apps/streamis/i18n/zh.json @@ -180,7 +180,8 @@ "choosedJar" : "选中的Jar包", "jarError": "文件必须是Jar包", "jarEmpty": "必须选择Jar包", - "jobNameEmpty": "作业名称不能为空" + "jobNameEmpty": "作业名称不能为空", + "choosedFile": "选中的文件" }, "projectFile": { "file": "文件", @@ -188,6 +189,7 @@ "overrideImport": "覆盖式导入", "chooseUploadFile": "请选择您要导入的资源文件", "versionPlaceholder": "只支持数字和.，例如0.1.0", + "commentLength": "文本长度过长,不能超过50个字符", "versionEmpty": "版本不能为空", "fileEmpty": "文件不能为空", "delete": "删除", diff --git a/web/src/apps/streamis/module/uploadFile/index.vue b/web/src/apps/streamis/module/uploadFile/index.vue index a78641a16..20e262d40 100644 --- a/web/src/apps/streamis/module/uploadFile/index.vue +++ b/web/src/apps/streamis/module/uploadFile/index.vue @@ -20,7 +20,7 @@ }}
- {{ $t('message.streamis.uploadJar.choosedJar') }}: {{ file.name }} + {{ $t('message.streamis.uploadJar.choosedFile') }}: {{ file.name }}
50){ + return callback(new Error(_this.$t('message.streamis.projectFile.commentLength'))) + }else{ + callback() + } + }, } } From f8dbed03c45c16ca0ad59f8fb26a74cc7f595148 Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 26 Oct 2022 16:32:13 +0800 Subject: [PATCH 039/219] Add module to collect spark container log. --- .../xspark-streamis-log-collector/pom.xml | 51 +++++++++++++++++ .../src/main/assembly/package.xml | 19 +++++++ .../spark/SparkStreamisConfigAutowired.java | 56 +++++++++++++++++++ ....jobmanager.plugin.StreamisConfigAutowired | 1 + streamis-jobmanager/streamis-job-log/pom.xml | 1 + .../log/entities/StreamisLogEvents.java | 1 + .../log/server/restful/JobLogRestfulApi.java | 4 ++ .../server/storage/StreamisJobLogStorage.java | 51 +++++++++-------- 8 files changed, 161 insertions(+), 23 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml new file mode 100644 index 000000000..6c2122698 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -0,0 +1,51 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + xspark-streamis-log-collector + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-log-collector1x + ${streamis.version} + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + + + assemble + + single + + + install + + + + + src/main/assembly/package.xml + + false + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml new file mode 100644 index 000000000..8da27bf2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml @@ -0,0 +1,19 @@ + + + package + + + jar + + false + + + / + true + runtime + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java new file mode 100644 index 000000000..a2a73adbe --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.spark; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; + +import java.util.Optional; + +/** + * Autoconfigure the streamis config in Spark environment + */ +public class SparkStreamisConfigAutowired implements StreamisConfigAutowired { + + private static final String APP_NAME_CONFIG = "app.name"; + + private static final String SERVER_ADDRESS_CONFIG = "streamis.url"; + + private static final String COLLECTOR_URI_CONFIG = "streamis.log.collector.uri"; + + private static final String PROJECT_NAME_CONFIG = "project.name"; + + private static final String DEFAULT_COLLECTOR_URI = "/api/rest_j/v1/streamis/streamJobManager/log/collect/events"; + @Override + public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception { + // Load the config from system properties + Optional.ofNullable(System.getProperty(APP_NAME_CONFIG)).ifPresent(appName -> { + String projectName = System.getProperty(PROJECT_NAME_CONFIG); + if (null != projectName && !projectName.trim().equals("")){ + appName = projectName + "." + appName; + } + System.out.println("Spark env to streamis: application name =>" + appName); + builder.setAppName(appName); + }); + String serverAddress = System.getProperty(SERVER_ADDRESS_CONFIG); + if (null != serverAddress && !serverAddress.trim().equals("")){ + if (serverAddress.endsWith("/")){ + serverAddress = serverAddress.substring(0, serverAddress.length() - 1); + } + String collectorUri = System.getProperty(COLLECTOR_URI_CONFIG, DEFAULT_COLLECTOR_URI); + if (null != collectorUri && !collectorUri.trim().equals("")){ + if (!collectorUri.startsWith("/")){ + collectorUri = "/" + collectorUri; + } + serverAddress += collectorUri; + } + System.out.println("Spark env to streamis: server address =>" + serverAddress); + builder.setRpcAddress(serverAddress); + } + String user = System.getenv("USER"); + if (null == user || user.trim().equals("")){ + user = System.getProperty("user.name", "hadoop"); + } + System.out.println("Spark env to streamis: log user =>" + user); + builder.setRpcAuthTokenUser(user); + return builder.build(); + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired new file mode 100644 index 000000000..dac2fcaed --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired @@ -0,0 +1 @@ +com.webank.wedatasphere.streamis.jobmanager.log.collector.spark.SparkStreamisConfigAutowired \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml index 0c8b087f1..eb79d84a5 100644 --- a/streamis-jobmanager/streamis-job-log/pom.xml +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -16,6 +16,7 @@ job-log-collector/streamis-job-log-collector job-log-collector/streamis-job-log-collector1x job-log-collector/flink-streamis-log-collector + job-log-collector/xspark-streamis-log-collector streamis-job-log-server streamis-job-log-common diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 7e942b675..624723a29 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -108,4 +108,5 @@ private String joinEvents(StreamisLogEvent[] events, String separator){ } return builder.toString(); } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java index 595a63e50..e9d19c651 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java @@ -1,9 +1,12 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.restful; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; import com.webank.wedatasphere.streamis.jobmanager.log.server.entities.StreamisLogEvents; import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; import com.webank.wedatasphere.streamis.jobmanager.log.server.service.StreamisJobLogService; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -15,6 +18,7 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; +import java.io.IOException; @RestController @RequestMapping(path = "/streamis/streamJobManager/log") diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index abe312b23..154c8f6f1 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -79,30 +79,35 @@ public JobLogBucket getOrCreateBucket(String userName, String appName, JobLogBuc public synchronized void init() { if (Objects.isNull(monitorThread)){ monitorThread = Utils.defaultScheduler().scheduleAtFixedRate(() -> { - Thread.currentThread().setName(StreamJobLogConfig.BUCKET_MONITOR_NAME.getValue()); - long maxIdleTime = StreamJobLogConfig.BUCKET_MAX_IDLE_TIME.getValue().toLong(); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - if (buckets.size() > 0) { - StringBuilder builder = new StringBuilder("Buckets in LogStorage: [\n"); - buckets.forEach((bucketName, bucket) -> { - JobLogBucketState bucketState = bucket.getBucketState(); - builder.append("bucket: [ name: ") - .append(bucketName) - .append(", path: ").append(bucketState.getBucketPath()) - .append(", parts: ").append(bucketState.getBucketParts()) - .append(", write-rate: ").append(bucketState.getBucketWriteRate()).append("/s") - .append(", last-write-time: ").append(dateFormat.format(bucketState.getBucketWriteTime())) - .append(" ]\n"); - if (bucketState.getBucketWriteTime() + maxIdleTime <= System.currentTimeMillis()){ - LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", - bucketName, dateFormat.format(bucketState.getBucketWriteTime())); - bucket.close(); - // Delete the bucket - buckets.remove(bucketName); - } + String threadName = Thread.currentThread().getName(); + try { + Thread.currentThread().setName(StreamJobLogConfig.BUCKET_MONITOR_NAME.getValue()); + long maxIdleTime = StreamJobLogConfig.BUCKET_MAX_IDLE_TIME.getValue().toLong(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + if (buckets.size() > 0) { + StringBuilder builder = new StringBuilder("Buckets in LogStorage: [\n"); + buckets.forEach((bucketName, bucket) -> { + JobLogBucketState bucketState = bucket.getBucketState(); + builder.append("bucket: [ name: ") + .append(bucketName) + .append(", path: ").append(bucketState.getBucketPath()) + .append(", parts: ").append(bucketState.getBucketParts()) + .append(", write-rate: ").append(bucketState.getBucketWriteRate()).append("/s") + .append(", last-write-time: ").append(dateFormat.format(bucketState.getBucketWriteTime())) + .append(" ]\n"); + if (bucketState.getBucketWriteTime() + maxIdleTime <= System.currentTimeMillis()) { + LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", + bucketName, dateFormat.format(bucketState.getBucketWriteTime())); + bucket.close(); + // Delete the bucket + buckets.remove(bucketName); + } - }); - LOG.info(builder.toString()); + }); + LOG.info(builder.toString()); + } + } finally { + Thread.currentThread().setName(threadName); } },BUCKET_MONITOR_INTERVAL.getValue().toLong(), BUCKET_MONITOR_INTERVAL.getValue().toLong(), TimeUnit.MILLISECONDS); From 8d006e1e0fd665b19f97fb67cb45a5e5c4775fc0 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 26 Oct 2022 17:26:04 +0800 Subject: [PATCH 040/219] Add the new requestBody parameter `projectName` for adding task and stop task for existed jobs. --- .../parser/SparkTaskMetricsParser.scala | 5 +- .../jobmanager/restful/api/JobRestfulApi.java | 207 +++++++++--------- 2 files changed, 105 insertions(+), 107 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala index a8e6bb7c3..c6e00d7ff 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -25,7 +25,10 @@ class SparkTaskMetricsParser extends AbstractTaskMetricsParser { val addDataNumberDTO: String => Unit = key => { val batch = new JobDetailsVo.DataNumberDTO batch.setDataName(key) - batch.setDataNumber(metricsMap.get(key).toString.toInt) + batch.setDataNumber(metricsMap.get(key) match { + case null => -1 + case num => num.toString.toInt + }) dataNumberDTOS.add(batch) } addDataNumberDTO("waitingBatchs") diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 7f2e396e3..e5796f14a 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -232,78 +232,88 @@ public Message executeHistoryJob(HttpServletRequest req, return Message.ok().data("details", details); } + private Message withStreamJob(HttpServletRequest req, String projectName, + String jobName, String username, + Function streamJobFunction) { + List streamJobs = streamJobService.getByProList(projectName, username, jobName, null, null).getList(); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to manage task."); + } + StreamJob streamJob = streamJobService.getJobById(streamJobs.get(0).getId()); + if (!streamJobService.hasPermission(streamJob, username) && + !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { + return Message.error("Have no permission to operate task for StreamJob [" + jobName + "]."); + } + return streamJobFunction.apply(streamJob); + } + @RequestMapping(path = "/addTask", method = RequestMethod.GET) public Message addTask(HttpServletRequest req, + @RequestParam(value = "projectName") String projectName, @RequestParam(value = "jobName") String jobName, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); - List streamJobs = streamJobService.getJobByName(jobName); - if(CollectionUtils.isEmpty(streamJobs)) { - return Message.error("Not exits Streamis job " + jobName); - } else if(streamJobs.size() > 1) { - return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); - } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to add new tasks."); - } - if (!streamJobService.hasPermission(streamJobs.get(0), username) && - !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { - return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); - } - // 如果存在正在运行的，先将其停止掉 - StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); - if(streamTask == null) { - // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo - // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 - PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, null, null); - List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark.")) - .collect(Collectors.toList()); - if(copyJobs.isEmpty()) { - return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); - } - int index = 0; - while(streamTask == null && index < copyJobs.size()) { - StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); - if(copyTask == null) { - index ++; - } else { - LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", - jobName, copyJobs.get(index).getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); - streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); - streamTask.setLinkisJobId(copyTask.getLinkisJobId()); - streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); - } - } + return withStreamJob(req, projectName, jobName, username, streamJob -> { + // 如果存在正在运行的，先将其停止掉 + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask == null) { - return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); - } - } else { - if(JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTask.setErrDesc("stopped by App's new task."); - streamTaskService.updateTask(streamTask); + // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo + // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 + PageInfo jobList = streamJobService.getByProList(streamJob.getProjectName(), username, null, null, null); + List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark.")) + .collect(Collectors.toList()); + if(copyJobs.isEmpty()) { + return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); + } + int index = 0; + while(streamTask == null && index < copyJobs.size()) { + StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); + if(copyTask == null) { + index ++; + } else { + LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", + jobName, copyJobs.get(index).getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); + streamTask = streamTaskService.createTask(streamJob.getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setLinkisJobId(copyTask.getLinkisJobId()); + streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); + } + } + if(streamTask == null) { + return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); + } + } else { + if(JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); + streamTaskService.updateTask(streamTask); + } + StreamTask newStreamTask = streamTaskService.createTask(streamJob.getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setId(newStreamTask.getId()); + streamTask.setVersion(newStreamTask.getVersion()); + streamTask.setErrDesc(""); + streamTask.setStatus(newStreamTask.getStatus()); + streamTask.setSubmitUser(username); } - StreamTask newStreamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); - streamTask.setId(newStreamTask.getId()); - streamTask.setVersion(newStreamTask.getVersion()); - streamTask.setErrDesc(""); - streamTask.setStatus(newStreamTask.getStatus()); - streamTask.setSubmitUser(username); - } - streamTask.setStartTime(new Date()); - streamTask.setLastUpdateTime(new Date()); - StreamTask finalStreamTask = streamTask; - return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { - flinkJobInfo.setApplicationId(appId); - flinkJobInfo.setApplicationUrl(appUrl); - flinkJobInfo.setName(jobName); - flinkJobInfo.setStatus(JobConf.getStatusString(finalStreamTask.getStatus())); - StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); - streamTaskService.updateTask(finalStreamTask); - LOG.info("Streamis Job {} has added a new task successfully.", jobName); - return Message.ok(); + streamTask.setStartTime(new Date()); + streamTask.setLastUpdateTime(new Date()); + StreamTask finalStreamTask = streamTask; + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + flinkJobInfo.setApplicationId(appId); + flinkJobInfo.setApplicationUrl(appUrl); + flinkJobInfo.setName(jobName); + flinkJobInfo.setStatus(JobConf.getStatusString(finalStreamTask.getStatus())); + StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); + streamTaskService.updateTask(finalStreamTask); + LOG.info("Streamis Job {} has added a new task successfully.", jobName); + return Message.ok(); + }); }); } @@ -320,67 +330,52 @@ private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Functio @RequestMapping(path = "/updateTask", method = RequestMethod.GET) public Message updateTask(HttpServletRequest req, + @RequestParam(value = "projectName") String projectName, @RequestParam(value = "jobName") String jobName, @RequestParam(value = "appId") String appId, @RequestParam(value = "metrics") String metrics) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to update task for Streamis job {} with appId: {}.", username, jobName, appId); - List streamJobs = streamJobService.getJobByName(jobName); - if(CollectionUtils.isEmpty(streamJobs)) { - return Message.error("Not exits Streamis job " + jobName); - } else if(streamJobs.size() > 1) { - return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); - } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to update task."); - } - StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); - if(streamTask == null) { - LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); - return Message.ok("not exists running task, ignore it."); - } else if(JobConf.isCompleted(streamTask.getStatus())) { - LOG.warn("The task of job {} is completed, ignore to update its metrics.", jobName); - return Message.ok("Task is completed, ignore to update its metrics."); - } - return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { - if(!flinkJobInfo.getApplicationId().equals(appId)) { - LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to update its metrics.", - jobName, flinkJobInfo.getApplicationId(), appId); - return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); - } - JobStateInfo jobStateInfo = new JobStateInfo(); - jobStateInfo.setTimestamp(System.currentTimeMillis()); - jobStateInfo.setLocation(metrics); - flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); - StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); - streamTaskService.updateTask(streamTask); - LOG.info("Streamis Job {} has updated the task metrics successfully.", jobName); - return Message.ok(); + return withStreamJob(req, projectName, jobName, username, streamJob -> { + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); + if (streamTask == null) { + LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); + return Message.ok("not exists running task, ignore it."); + } else if (JobConf.isCompleted(streamTask.getStatus())) { + LOG.warn("The task of job {} is completed, ignore to update its metrics.", jobName); + return Message.ok("Task is completed, ignore to update its metrics."); + } + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if (!flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to update its metrics.", + jobName, flinkJobInfo.getApplicationId(), appId); + return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); + } + JobStateInfo jobStateInfo = new JobStateInfo(); + jobStateInfo.setTimestamp(System.currentTimeMillis()); + jobStateInfo.setLocation(metrics); + flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); + StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); + streamTaskService.updateTask(streamTask); + LOG.info("Streamis Job {} has updated the task metrics successfully.", jobName); + return Message.ok(); + }); }); } @RequestMapping(path = "/stopTask", method = RequestMethod.GET) public Message stopTask(HttpServletRequest req, + @RequestParam(value = "projectName") String projectName, @RequestParam(value = "jobName") String jobName, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); - List streamJobs = streamJobService.getJobByName(jobName); - if(CollectionUtils.isEmpty(streamJobs)) { - return Message.error("Not exits Streamis job " + jobName); - } else if(streamJobs.size() > 1) { - return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); - } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to stop task."); - } - return tryStopTask(req, username, streamJobs.get(0), appId); + return withStreamJob(req, projectName, jobName, username, + streamJob -> tryStopTask(req, username, streamJob, appId)); } private Message tryStopTask(HttpServletRequest req, String username, StreamJob streamJob, String appId) { - if (!streamJobService.hasPermission(streamJob, username) && - !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { - return Message.error("Have no permission to stop task for StreamJob [" + streamJob.getName() + "]."); - } // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { From c518e10827d9c5b62ab727c9d7ee18aab9ca0475 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 26 Oct 2022 17:28:40 +0800 Subject: [PATCH 041/219] Add the new requestBody parameter `projectName` for adding task and stop task for existed jobs. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index e5796f14a..1a7cbc593 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -190,7 +190,7 @@ public Message killJob(HttpServletRequest req, } } else { LOG.error("{} try to kill not-supported-management job {} with name {}.", userName, jobId, streamJob.getName()); - return tryStopTask(req, userName, streamJob, null); + return tryStopTask(streamJob, null); } } @@ -372,10 +372,10 @@ public Message stopTask(HttpServletRequest req, String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); return withStreamJob(req, projectName, jobName, username, - streamJob -> tryStopTask(req, username, streamJob, appId)); + streamJob -> tryStopTask(streamJob, appId)); } - private Message tryStopTask(HttpServletRequest req, String username, StreamJob streamJob, String appId) { + private Message tryStopTask(StreamJob streamJob, String appId) { // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { From 49c1f952b5ed97dfd8400200f5b412806c6ad9ec Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 27 Oct 2022 02:48:23 +0800 Subject: [PATCH 042/219] Complete the strategy in JsonTool.escapeStrValue(). --- .../log/entities/StreamisLogEvent.java | 2 +- .../log/entities/StreamisLogEvents.java | 2 +- .../jobmanager/log/json/JsonTool.java | 22 +++++++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index fe340e066..6f8645f77 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -76,7 +76,7 @@ public void setSequenceId(int sequenceId){ public String toJson(){ return "{" + "\"logTimeStamp\":" + logTimeInMills + - ",\"content\":" + (Objects.isNull(content)? null : "\"" + JsonTool.encodeStrValue(content) + "\"") + + ",\"content\":" + (Objects.isNull(content)? null : "\"" + JsonTool.escapeStrValue(content) + "\"") + ",\"sequenceId\":0" + "}"; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 624723a29..f2843c8af 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -91,7 +91,7 @@ public void setSequenceId(int sequenceId){ public String toJson(){ return "{" + "\"logTimeStamp\":" + logTimeInMills + - ",\"appName\":" + (Objects.isNull(appName)? null : "\"" + JsonTool.encodeStrValue(appName) + "\"") + + ",\"appName\":" + (Objects.isNull(appName)? null : "\"" + JsonTool.escapeStrValue(appName) + "\"") + ",\"events\":[" + (Objects.isNull(events) || events.length <=0 ? "" : joinEvents(events, ",") ) + "]" + ",\"sequenceId\":0" diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java index d59749261..0822e820d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java @@ -1,13 +1,16 @@ package com.webank.wedatasphere.streamis.jobmanager.log.json; +import java.util.Locale; + public class JsonTool { + static final char[] HEX_DIGITS = new char[] {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; /** * Avoid the special char * @param input input string * @return output string */ - public static String encodeStrValue(String input){ + public static String escapeStrValue(String input){ char[] chars = input.toCharArray(); StringBuilder sb = new StringBuilder(); for (char c : chars) { @@ -37,9 +40,24 @@ public static String encodeStrValue(String input){ sb.append("\\t"); break; default: - sb.append(c); + sb.append((c < 32) ? escapeUnicode(c) : c); } } return sb.toString(); } + + /** + * Escape unicode + * @param code char code + * @return escaped string + */ + private static String escapeUnicode(int code){ + if (code > 0xffff){ + return "\\u" + Integer.toHexString(code).toUpperCase(Locale.ENGLISH); + } else { + return "\\u" + HEX_DIGITS[(code >> 12) & 15] + + HEX_DIGITS[(code >> 8) & 15] + HEX_DIGITS[(code >> 4) & 15] + HEX_DIGITS[code & 15]; + } + } + } From b026ee26fba02ef7db148e9d0e0d7005cf2794ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Fri, 28 Oct 2022 17:43:49 +0800 Subject: [PATCH 043/219] =?UTF-8?q?upd:=20=E8=BF=9B=E5=85=A5=E9=A6=96?= =?UTF-8?q?=E9=A1=B5=E6=97=B6=E8=8B=A5projectName=E6=9C=AA=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E5=88=99=E4=BB=8Elocation=20search=E4=B8=AD=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=90=8E=E8=B7=AF=E7=94=B1=E5=88=B0=E5=AF=B9=E5=BA=94?= =?UTF-8?q?=E7=9A=84=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/view/realTimeJobCenter/index.vue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/src/apps/streamis/view/realTimeJobCenter/index.vue b/web/src/apps/streamis/view/realTimeJobCenter/index.vue index 56f859270..31c093739 100644 --- a/web/src/apps/streamis/view/realTimeJobCenter/index.vue +++ b/web/src/apps/streamis/view/realTimeJobCenter/index.vue @@ -16,10 +16,14 @@ export default { data() { return { navHeight: 0, + projectName: this.$route.query.projectName || (new URLSearchParams(window.location.search)).get('projectName') }; }, mounted() { // this.init(); + if(!this.$route.query.projectName){ + this.$router.push(`/realTimeJobCenter?projectName=${this.projectName}`) + } }, methods: { resize(height) { From d882cf4444eb0e1c04037ad4b15547b687a4db7c Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Sat, 29 Oct 2022 15:56:58 +0800 Subject: [PATCH 044/219] resolve 413 response status for update task. --- .../jobmanager/restful/api/JobRestfulApi.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 1a7cbc593..a9f187efe 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -328,14 +328,15 @@ private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Functio return flinkJobInfoFunction.apply(flinkJobInfo); } - @RequestMapping(path = "/updateTask", method = RequestMethod.GET) + @RequestMapping(path = "/updateTask", method = RequestMethod.POST) public Message updateTask(HttpServletRequest req, - @RequestParam(value = "projectName") String projectName, - @RequestParam(value = "jobName") String jobName, - @RequestParam(value = "appId") String appId, - @RequestParam(value = "metrics") String metrics) { + @RequestBody Map json) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to update task for Streamis job {} with appId: {}.", username, jobName, appId); + String projectName = json.get("projectName"); + String jobName = json.get("jobName"); + String appId = json.get("appId"); + String metrics = json.get("metrics"); + LOG.info("User {} try to update task for Streamis job {} with appId: {}, metrics: {}.", username, jobName, appId, metrics); return withStreamJob(req, projectName, jobName, username, streamJob -> { StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if (streamTask == null) { From d33b0f33312c6dde420dce3901e6f3ae6132ca53 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Sat, 29 Oct 2022 16:07:51 +0800 Subject: [PATCH 045/219] resolve 413 response status for update task. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index a9f187efe..5eff10874 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -235,6 +235,11 @@ public Message executeHistoryJob(HttpServletRequest req, private Message withStreamJob(HttpServletRequest req, String projectName, String jobName, String username, Function streamJobFunction) { + if(StringUtils.isBlank(projectName)) { + return Message.error("projectName cannot be empty!"); + } else if(StringUtils.isBlank(jobName)) { + return Message.error("jobName cannot be empty!"); + } List streamJobs = streamJobService.getByProList(projectName, username, jobName, null, null).getList(); if(CollectionUtils.isEmpty(streamJobs)) { return Message.error("Not exits Streamis job " + jobName); @@ -259,6 +264,9 @@ public Message addTask(HttpServletRequest req, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + if(StringUtils.isBlank(appId)) { + return Message.error("appId cannot be empty!"); + } return withStreamJob(req, projectName, jobName, username, streamJob -> { // 如果存在正在运行的，先将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); From 11cf7c72fbc183fece6916a229a6c4c4e80bb6f7 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sat, 29 Oct 2022 16:13:01 +0800 Subject: [PATCH 046/219] Fix the in method restoreInvariants of AbstractRpcLogSender. --- .../jobmanager/log/collector/sender/AbstractRpcLogSender.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index d90eb1a7f..3afc34147 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -435,7 +435,8 @@ private void restoreInvariants(int i, int take, boolean clearItems){ this.count -= i; if (clearItems){ int index = this.takeIndex; - for (; i > 0; i --){ + int j = i; + for (; j > 0; j --){ this.items[index] = null; if (++index == items.length){ index = 0; From e9c8792aaa19985596cd3043d51e00e0b4a30f1a Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Mon, 31 Oct 2022 16:11:12 +0800 Subject: [PATCH 047/219] resolve 413 response status for update task. --- .../jobmanager/restful/api/JobRestfulApi.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 5eff10874..44e1bdcc6 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -336,14 +336,13 @@ private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Functio return flinkJobInfoFunction.apply(flinkJobInfo); } - @RequestMapping(path = "/updateTask", method = RequestMethod.POST) + @RequestMapping(path = "/updateTask", method = RequestMethod.GET) public Message updateTask(HttpServletRequest req, - @RequestBody Map json) { + @RequestParam(value = "projectName") String projectName, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "metrics") String metrics) { String username = SecurityFilter.getLoginUsername(req); - String projectName = json.get("projectName"); - String jobName = json.get("jobName"); - String appId = json.get("appId"); - String metrics = json.get("metrics"); LOG.info("User {} try to update task for Streamis job {} with appId: {}, metrics: {}.", username, jobName, appId, metrics); return withStreamJob(req, projectName, jobName, username, streamJob -> { StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); @@ -372,6 +371,16 @@ public Message updateTask(HttpServletRequest req, }); } + @RequestMapping(path = "/updateTask", method = RequestMethod.POST) + public Message updateTask(HttpServletRequest req, + @RequestBody Map json) { + String projectName = json.get("projectName"); + String jobName = json.get("jobName"); + String appId = json.get("appId"); + String metrics = json.get("metrics"); + return updateTask(req, projectName, jobName, appId, metrics); + } + @RequestMapping(path = "/stopTask", method = RequestMethod.GET) public Message stopTask(HttpServletRequest req, @RequestParam(value = "projectName") String projectName, From b3d60fe5524aa695d52b949824d123501d358228 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 7 Nov 2022 11:31:12 +0800 Subject: [PATCH 048/219] adjust the relation path in pom.xml. --- .../job-log-collector/flink-streamis-log-collector/pom.xml | 1 + .../job-log-collector/streamis-job-log-collector-core/pom.xml | 1 + .../job-log-collector/streamis-job-log-collector/pom.xml | 1 + .../job-log-collector/streamis-job-log-collector1x/pom.xml | 1 + .../job-log-collector/xspark-streamis-log-collector/pom.xml | 1 + .../streamis-job-log/streamis-job-log-common/pom.xml | 1 + .../streamis-job-log/streamis-job-log-server/pom.xml | 1 + 7 files changed, 7 insertions(+) diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml index 6bbee8ba8..48625dbcb 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml index bc68d0796..b8ac1ed54 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index e2e556fc8..bca7178c3 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml index 43792639c..950ec38fb 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml index 6c2122698..13ef5b0e2 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index 4a171b938..1eb4cdb1e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index ec75cea68..9facffe7d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../pom.xml 4.0.0 From 67defcea220f11e82f997f7b349c93afa72fec18 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 7 Nov 2022 19:31:11 +0800 Subject: [PATCH 049/219] Fix the problem in pom.xml of stream job log modules. --- .../job-log-collector/flink-streamis-log-collector/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector-core/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector1x/pom.xml | 2 +- .../job-log-collector/xspark-streamis-log-collector/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-common/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-server/pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml index 48625dbcb..ca49e4399 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml index b8ac1ed54..0cdb47b8d 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index bca7178c3..55fc60818 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml index 950ec38fb..7632a5718 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml index 13ef5b0e2..904bec4dd 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index 1eb4cdb1e..fcff3592c 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../pom.xml + ../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 9facffe7d..165f6a520 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../pom.xml + ../pom.xml 4.0.0 From 02dcdc6d1a60597d110d3b804c0eaa4d4b55c14b Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 15 Nov 2022 17:21:13 +0800 Subject: [PATCH 050/219] resolve the LinkisJobInfo is null for adding task. --- .../jobmanager/restful/api/JobRestfulApi.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 44e1bdcc6..bb84d42f7 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -263,14 +263,20 @@ public Message addTask(HttpServletRequest req, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + LOG.info("User {} try to add a new task for Streamis job {}.{} with appId: {}, appUrl: {}.", username, projectName, jobName, appId, appUrl); if(StringUtils.isBlank(appId)) { return Message.error("appId cannot be empty!"); } return withStreamJob(req, projectName, jobName, username, streamJob -> { // 如果存在正在运行的，先将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); - if(streamTask == null) { + if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); + streamTaskService.updateTask(streamTask); + } + if(streamTask == null || StringUtils.isBlank(streamTask.getLinkisJobInfo())) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 PageInfo jobList = streamJobService.getByProList(streamJob.getProjectName(), username, null, null, null); @@ -280,9 +286,10 @@ public Message addTask(HttpServletRequest req, return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); } int index = 0; + streamTask = null; while(streamTask == null && index < copyJobs.size()) { StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); - if(copyTask == null) { + if(copyTask == null || StringUtils.isBlank(copyTask.getLinkisJobInfo())) { index ++; } else { LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", @@ -296,12 +303,6 @@ public Message addTask(HttpServletRequest req, return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); } } else { - if(JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTask.setErrDesc("stopped by App's new task."); - streamTaskService.updateTask(streamTask); - } StreamTask newStreamTask = streamTaskService.createTask(streamJob.getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); streamTask.setId(newStreamTask.getId()); streamTask.setVersion(newStreamTask.getVersion()); @@ -343,7 +344,7 @@ public Message updateTask(HttpServletRequest req, @RequestParam(value = "appId") String appId, @RequestParam(value = "metrics") String metrics) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to update task for Streamis job {} with appId: {}, metrics: {}.", username, jobName, appId, metrics); + LOG.info("User {} try to update task for Streamis job {}.{} with appId: {}, metrics: {}.", username, projectName, jobName, appId, metrics); return withStreamJob(req, projectName, jobName, username, streamJob -> { StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if (streamTask == null) { @@ -388,7 +389,7 @@ public Message stopTask(HttpServletRequest req, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + LOG.info("User {} try to stop task for Streamis job {}.{} with appId: {}, appUrl: {}.", username, projectName, jobName, appId, appUrl); return withStreamJob(req, projectName, jobName, username, streamJob -> tryStopTask(streamJob, appId)); } From 38b3742b897eea0b533f3209493c6394ec93aae2 Mon Sep 17 00:00:00 2001 From: jefftlin <57786627+jefftlin@users.noreply.github.com> Date: Tue, 20 Dec 2022 11:30:07 +0800 Subject: [PATCH 051/219] Upgrade the version of pom to 0.2.3 (#55) * Upgrade the version of pom to 0.2.3 * Minor fix Co-authored-by: jefftlin --- assembly/pom.xml | 2 +- pom.xml | 6 +++--- streamis-appconn/pom.xml | 2 +- streamis-jobmanager/pom.xml | 2 +- streamis-jobmanager/streamis-job-launcher/pom.xml | 2 +- .../streamis-job-launcher-base/pom.xml | 2 +- .../streamis-job-launcher-linkis/pom.xml | 2 +- .../streamis-job-launcher-service/pom.xml | 2 +- .../job-log-collector/flink-streamis-log-collector/pom.xml | 2 +- .../streamis-job-log-collector-core/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector1x/pom.xml | 2 +- .../job-log-collector/xspark-streamis-log-collector/pom.xml | 2 +- streamis-jobmanager/streamis-job-log/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-common/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-server/pom.xml | 4 ++-- streamis-jobmanager/streamis-job-manager/pom.xml | 2 +- .../streamis-job-manager/streamis-job-manager-base/pom.xml | 2 +- .../streamis-job-manager-service/pom.xml | 2 +- streamis-jobmanager/streamis-jobmanager-common/pom.xml | 2 +- streamis-jobmanager/streamis-jobmanager-server/pom.xml | 2 +- streamis-jobmanager/streamis-projectmanager-server/pom.xml | 2 +- streamis-project/pom.xml | 2 +- streamis-project/streamis-project-common/pom.xml | 2 +- streamis-project/streamis-project-server/pom.xml | 2 +- streamis-server/pom.xml | 2 +- web/package.json | 2 +- 27 files changed, 30 insertions(+), 30 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 8f8038bbc..2b788a362 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/pom.xml b/pom.xml index 0fc4d2fba..93c1a1bbd 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ com.webank.wedatasphere.streamis streamis - 0.2.0 + 0.2.3 pom Streamis Project Parent POM @@ -48,7 +48,7 @@ 1.1.3 4.12 1.1.0 - 0.2.0 + 0.2.3 2.11.12 1.8 3.3.3 @@ -70,7 +70,7 @@ 2.21 1.9.5 1.4.19 - 0.2.0 + 0.2.3 5.1.47 2.0.1.Final diff --git a/streamis-appconn/pom.xml b/streamis-appconn/pom.xml index 54b590f91..4b4a38caa 100644 --- a/streamis-appconn/pom.xml +++ b/streamis-appconn/pom.xml @@ -5,7 +5,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/pom.xml b/streamis-jobmanager/pom.xml index e88713dc2..eec37a4a5 100644 --- a/streamis-jobmanager/pom.xml +++ b/streamis-jobmanager/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/pom.xml b/streamis-jobmanager/streamis-job-launcher/pom.xml index 8964504e1..de440be80 100755 --- a/streamis-jobmanager/streamis-job-launcher/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml index 00f62af34..542447a01 100755 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml index fdd13095d..01d9ec0f2 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml @@ -20,7 +20,7 @@ streamis-job-launcher com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml index 545888044..278af5594 100755 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml index ca49e4399..1cd05a3a1 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml index 0cdb47b8d..fa0785351 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index 55fc60818..098ccf214 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml index 7632a5718..2956d8bf9 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml index 904bec4dd..cdd5d26ed 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml index eb79d84a5..f45e271e1 100644 --- a/streamis-jobmanager/streamis-job-log/pom.xml +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -5,7 +5,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index fcff3592c..b3429ac9e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 165f6a520..6a6454c12 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.streamis streamis-job-log-common - 0.2.0 + 0.2.3 org.apache.linkis diff --git a/streamis-jobmanager/streamis-job-manager/pom.xml b/streamis-jobmanager/streamis-job-manager/pom.xml index 8c80a3520..9d2ae3ae0 100755 --- a/streamis-jobmanager/streamis-job-manager/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml index f0ac3a5c2..b4e343e6b 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml index aec49b65b..cd3814726 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-jobmanager-common/pom.xml b/streamis-jobmanager/streamis-jobmanager-common/pom.xml index b63c59d15..f6c322730 100644 --- a/streamis-jobmanager/streamis-jobmanager-common/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-common/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-jobmanager-server/pom.xml b/streamis-jobmanager/streamis-jobmanager-server/pom.xml index 11f353a55..cbf51250f 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-server/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-projectmanager-server/pom.xml b/streamis-jobmanager/streamis-projectmanager-server/pom.xml index c5066a706..e6ca1ba2e 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-projectmanager-server/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-project/pom.xml b/streamis-project/pom.xml index e5bbcf888..6d5ad7295 100644 --- a/streamis-project/pom.xml +++ b/streamis-project/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-project/streamis-project-common/pom.xml b/streamis-project/streamis-project-common/pom.xml index 849710b20..9d6d1c9f1 100644 --- a/streamis-project/streamis-project-common/pom.xml +++ b/streamis-project/streamis-project-common/pom.xml @@ -20,7 +20,7 @@ streamis-project com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-project/streamis-project-server/pom.xml b/streamis-project/streamis-project-server/pom.xml index 880d5bf0b..0833f2c6a 100644 --- a/streamis-project/streamis-project-server/pom.xml +++ b/streamis-project/streamis-project-server/pom.xml @@ -20,7 +20,7 @@ streamis-project com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-server/pom.xml b/streamis-server/pom.xml index 1bf8ecfba..5ab3784ba 100644 --- a/streamis-server/pom.xml +++ b/streamis-server/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/web/package.json b/web/package.json index fc5a8bfb2..598101ac9 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "streamis", - "version": "0.2.0", + "version": "0.2.3", "private": true, "scripts": { "serve": "vue-cli-service serve", From 64ae463d2ab458a3d55d73ee1aefdf234d83acce Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 20 Dec 2022 16:56:27 +0800 Subject: [PATCH 052/219] Use the "flink.yarn.ship-directories" instead of "flink.app.user.class.path" fixed #56 --- .../FlinkJarStreamisJobContentTransform.scala | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala index 551ae0f98..72206eee0 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala @@ -27,6 +27,7 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.utils.JobUtils import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer /** * Created by enjoyyin on 2021/9/23. @@ -50,20 +51,24 @@ class FlinkJarStreamisStartupParamsTransform extends Transform { startupMap.put("flink.app.main.class.jar", transformJobContent.getMainClassJar.getFileName) startupMap.put("flink.app.main.class.jar.bml.json", JsonUtils.jackson.writeValueAsString(getStreamisFileContent(transformJobContent.getMainClassJar))) - val classpathFiles = if(transformJobContent.getDependencyJars != null && transformJobContent.getResources != null) { - startupMap.put("flink.app.user.class.path", transformJobContent.getDependencyJars.asScala.map(_.getFileName).mkString(",")) - transformJobContent.getDependencyJars.asScala ++ transformJobContent.getResources.asScala - } else if(transformJobContent.getDependencyJars != null) { - startupMap.put("flink.app.user.class.path", transformJobContent.getDependencyJars.asScala.map(_.getFileName).mkString(",")) - transformJobContent.getDependencyJars.asScala - } else if(transformJobContent.getResources != null) { - startupMap.put("flink.yarn.ship-directories", transformJobContent.getResources.asScala.map(_.getFileName).mkString(",")) - transformJobContent.getResources.asScala + + /** + * Notice : "flink.app.user.class.path" equals to PipelineOptions.CLASSPATHS in Flink + * paths must specify a protocol (e.g. file://) and be accessible on all nodes + * so we use "flink.yarn.ship-directories" instead + */ + var classPathFiles = Option(transformJobContent.getDependencyJars) match { + case Some(list) => list.asScala + case _ => mutable.Buffer[StreamisFile]() + } + Option(transformJobContent.getResources) match { + case Some(list) => classPathFiles = classPathFiles ++ list.asScala + case _ => // Do nothing } - else mutable.Buffer[StreamisFile]() - if(classpathFiles.nonEmpty) + startupMap.put("flink.yarn.ship-directories", classPathFiles.map(_.getFileName).mkString(",")) + if(classPathFiles.nonEmpty) startupMap.put("flink.app.user.class.path.bml.json", - JsonUtils.jackson.writeValueAsString(classpathFiles.map(getStreamisFileContent).asJava)) + JsonUtils.jackson.writeValueAsString(classPathFiles.map(getStreamisFileContent).asJava)) if(transformJobContent.getHdfsJars != null) startupMap.put("flink.user.lib.path", transformJobContent.getHdfsJars.asScala.mkString(",")) val params = if(job.getParams == null) new util.HashMap[String, Any] else job.getParams From 136a02900dacd08ca4148c1ca7df1dcb9e5b105f Mon Sep 17 00:00:00 2001 From: liveipooli Date: Tue, 20 Dec 2022 17:55:23 +0800 Subject: [PATCH 053/219] add: features --- web/src/apps/streamis/i18n/en.json | 4 +++ web/src/apps/streamis/i18n/zh.json | 4 +++ .../apps/streamis/module/jobList/index.vue | 27 ++++++++++++++++++- .../apps/streamis/module/jobSummary/index.vue | 16 ++++++++++- .../streamis/module/versionDetail/index.vue | 4 +++ web/vue.config.js | 4 +-- 6 files changed, 55 insertions(+), 4 deletions(-) diff --git a/web/src/apps/streamis/i18n/en.json b/web/src/apps/streamis/i18n/en.json index fad63b15f..0fdd35f53 100644 --- a/web/src/apps/streamis/i18n/en.json +++ b/web/src/apps/streamis/i18n/en.json @@ -56,6 +56,8 @@ "jobListTableColumns":{ "jobName": "Job name", "jobType": "Job type", + "launchMode": "Launch Mode", + "manageMode": "Manage Mode", "taskStatus": "Task status", "lastReleaseTime": "Last release time", "label": "Label", @@ -78,6 +80,8 @@ "modalTitle":"Version detail", "jobId":"Job id", "version":"Version", + "using":"Using", + "versionStatus":"VersionStatus", "description":"Description", "createTime":"Create time", "creator":"Creator", diff --git a/web/src/apps/streamis/i18n/zh.json b/web/src/apps/streamis/i18n/zh.json index 13eccbe64..834a3ff31 100644 --- a/web/src/apps/streamis/i18n/zh.json +++ b/web/src/apps/streamis/i18n/zh.json @@ -55,6 +55,8 @@ "jobListTableColumns":{ "jobName": "作业名称", "jobType": "作业类型", + "launchMode": "运行模式", + "manageMode": "管理模式", "taskStatus": "运行状态", "lastReleaseTime": "最近发布时间", "label": "标签", @@ -77,6 +79,8 @@ "modalTitle":"版本详情", "jobId":"作业id", "version":"版本", + "using":"使用中", + "versionStatus":"版本状态", "description":"描述", "createTime":"创建时间", "creator":"创建者", diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 23ea4bd5d..eaa7cd9d8 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -130,6 +130,7 @@
{{ row.version }}
+
{{row.versionForwards > 0 ? '(+' + row.versionForwards + ')' : ''}}
- + diff --git a/web/src/apps/streamis/module/versionDetail/index.vue b/web/src/apps/streamis/module/versionDetail/index.vue index ba455b640..db678f298 100644 --- a/web/src/apps/streamis/module/versionDetail/index.vue +++ b/web/src/apps/streamis/module/versionDetail/index.vue @@ -5,6 +5,7 @@ v-model="visible" footer-hide width="1200" + class="version-detail-modal" @on-cancel="cancel" > @@ -95,3 +96,11 @@ export default { } + From fb5e4e21f13d81d11b3cc23243ab8e29075b4a00 Mon Sep 17 00:00:00 2001 From: Liveipool <123653079@qq.com> Date: Thu, 9 Feb 2023 16:28:04 +0800 Subject: [PATCH 093/219] upd: projectName --- web/src/apps/streamis/module/jobList/index.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 4a9244903..2144a8aad 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -511,11 +511,11 @@ export default { pageNow: current, pageSize, // 本地开发dev环境用的 - projectName: 'stream_job', + // projectName: 'stream_job', // 本地开发sit环境用的 // projectName: 'streamis025_version', // 正式环境用的 - // projectName: this.projectName + projectName: this.projectName } const { jobName, jobStatus } = this.query if (jobName) { From d4789a6046cc2cb8f41d98802f32f0a60af54181 Mon Sep 17 00:00:00 2001 From: Liveipool <123653079@qq.com> Date: Thu, 9 Feb 2023 17:25:26 +0800 Subject: [PATCH 094/219] add: pagination --- .../apps/streamis/module/jobList/index.vue | 64 +++++++------------ .../streamis/module/versionDetail/index.vue | 25 +++++++- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 2144a8aad..93cf7d6be 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -204,10 +204,12 @@ 0) { - api.fetch(`streamis/streamJobManager/job/${data.id}/versions?pageNow=${1}&pageSize=${100}`, 'get').then(res => { - console.log('versionDetail versions res: ', res); - this.loading = false - this.modalVisible = true - this.versionDatas = res.versions - this.versionDatas.forEach(item => { - item.versionStatus = '--' - if (item.version === data.version) item.versionStatus = this.$t('message.streamis.versionDetail.using') - }) - }).catch(err => { - console.log('versionDetail versions err: ', err); - this.loading = false + api.fetch(`streamis/streamJobManager/job/${data.id}/versions?pageNow=${page ? page : 1}&pageSize=5`, 'get').then(res => { + console.log('versionDetail versions res: ', res); + this.loading = false + this.modalVisible = true + this.$refs.versionDetail.pageData.total = res.totalPage + this.versionDatas = res.versions + this.versionDatas.forEach(item => { + item.versionStatus = '--' + if (item.version === data.version) item.versionStatus = this.$t('message.streamis.versionDetail.using') }) - } else { - api - .fetch( - 'streamis/streamJobManager/job/version?jobId=' + - data.id + - '&version=' + - data.version, - 'get' - ) - .then(res => { - console.log('versionDetail version res: ', res); - if (res) { - this.loading = false - this.modalVisible = true - this.versionDatas = [res.detail] - this.versionDatas.forEach(item => { - item.versionStatus = '--' - if (item.version === data.version) item.versionStatus = this.$t('message.streamis.versionDetail.using') - }) - } - }) - .catch(err => { - console.log('versionDetail version err: ', err); - this.loading = false - }) - } + }).catch(err => { + console.log('versionDetail versions err: ', err); + this.loading = false + }) }, modalCancel() { this.modalVisible = false diff --git a/web/src/apps/streamis/module/versionDetail/index.vue b/web/src/apps/streamis/module/versionDetail/index.vue index db678f298..7340ff4de 100644 --- a/web/src/apps/streamis/module/versionDetail/index.vue +++ b/web/src/apps/streamis/module/versionDetail/index.vue @@ -20,6 +20,14 @@
+ @@ -35,6 +43,10 @@ export default { }, data() { return { + pageData: { + total: 0, + current: 1 + }, columns: [ { title: this.$t('message.streamis.versionDetail.jobId'), @@ -91,11 +103,20 @@ export default { }, cancel() { this.$emit('modalCancel') - } + }, + handlePageChange(page) { + this.pageData.current = page + this.$emit('versionDetailPageChange', page) + }, } } - + + + \ No newline at end of file diff --git a/web/vue.config.js b/web/vue.config.js index 5bd80a7dc..b77872252 100644 --- a/web/vue.config.js +++ b/web/vue.config.js @@ -124,8 +124,8 @@ module.exports = { port: 8080, proxy: { '/api': { - target: 'http://sit.dss.bdp.weoa.com', - // target: 'http://10.107.97.166:9188', + // target: 'http://sit.dss.bdp.weoa.com', + target: 'http://10.107.97.166:9188', // target: 'http://172.24.2.230:9400', changeOrigin: true, pathRewrite: { From 591d9ef01510905e26148817cc32eaac8140011d Mon Sep 17 00:00:00 2001 From: alexkun Date: Wed, 24 May 2023 18:41:38 +0800 Subject: [PATCH 162/219] streamis-job-manager-service - merge getByProList --- .../jobmanager/manager/service/DefaultStreamJobService.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index 44cce85fc..acf9d243a 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -65,10 +65,8 @@ class DefaultStreamJobService extends StreamJobService with Logging { override def getJobByName(jobName: String): util.List[StreamJob] = streamJobMapper.getJobByName(jobName) - override def getByProList(projectName: String, userName: String, jobName: String, jobStatus: Integer, jobCreator: String): PageInfo[QueryJobListVo] = { - val streamJobList = streamJobMapper.getJobLists(projectName, userName, jobName, jobStatus, jobCreator, JobConfKeyConstants.MANAGE_MODE_KEY.getValue) override def getByProList(projectName: String, userName: String, jobName: String, jobStatus: Integer, jobCreator: String, label: String): PageInfo[QueryJobListVo] = { - val streamJobList = streamJobMapper.getJobLists(projectName, userName, jobName, jobStatus, jobCreator,label) + val streamJobList = streamJobMapper.getJobLists(projectName, userName, jobName, jobStatus, jobCreator, label, JobConfKeyConstants.MANAGE_MODE_KEY.getValue) if (streamJobList != null && !streamJobList.isEmpty) { val pageInfo = new PageInfo[QueryJobListVo](streamJobList) return pageInfo From 0d2631f7f3a29f31233707ffe10c7a0237cbcef7 Mon Sep 17 00:00:00 2001 From: alexkun Date: Thu, 25 May 2023 12:57:06 +0800 Subject: [PATCH 163/219] fix import --- .../jobmanager/manager/service/DefaultStreamJobService.scala | 4 +--- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index acf9d243a..6b973bb3d 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -23,11 +23,9 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.launcher.job.exception.{JobCreateErrorException, JobFetchErrorException} import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService import com.webank.wedatasphere.streamis.jobmanager.manager.alert.AlertLevel -import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamAlertMapper, StreamJobMapper, StreamTaskMapper} import com.webank.wedatasphere.streamis.jobmanager.manager.entity._ import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{QueryJobListVo, TaskCoreNumVo, VersionDetailVo} -import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobCreateErrorException, JobFetchErrorException} import com.webank.wedatasphere.streamis.jobmanager.manager.service.DefaultStreamJobService.JobDeployValidateResult import com.webank.wedatasphere.streamis.jobmanager.manager.transform.JobContentParser import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent @@ -91,7 +89,7 @@ class DefaultStreamJobService extends StreamJobService with Logging { * COre indicator(核心指标) */ override def countByCores(projectName: String, userName: String): TaskCoreNumVo = { - val jobs = streamJobMapper.getJobLists(projectName, userName, null, null, null, JobConfKeyConstants.MANAGE_MODE_KEY.getValue) + val jobs = streamJobMapper.getJobLists(projectName, userName, null, null, null, null, JobConfKeyConstants.MANAGE_MODE_KEY.getValue) val taskNum = new TaskCoreNumVo() taskNum.setProjectName(projectName) if (jobs != null && !jobs.isEmpty) { diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index cb2d682e3..7e575bc54 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -21,12 +21,12 @@ import com.webank.wedatasphere.streamis.jobmanager.exception.JobException; import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.JobInfo; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobStateInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.jobInfo.EngineConnJobInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService; -import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.MetaJsonInfo; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; From 0448d9734a50562320c32dc101162d8877fa7f37 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Thu, 25 May 2023 14:27:27 +0800 Subject: [PATCH 164/219] parameter modify --- .../jobmanager/manager/service/DefaultStreamJobService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index b5ef0d21b..c531faa9c 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -313,7 +313,7 @@ class DefaultStreamJobService extends StreamJobService with Logging { } - override def updateLabel(streamJob: StreamJob): Unit = streamJobMapper.updateJob(StreamJob) + override def updateLabel(streamJob: StreamJob): Unit = streamJobMapper.updateJob(streamJob) } From ae297065a08aa4f44d1bd421dc21dc96ec50b35a Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Thu, 25 May 2023 15:33:23 +0800 Subject: [PATCH 165/219] add regular check --- .../jobmanager/restful/api/JobRestfulApi.java | 2 ++ .../streamis/jobmanager/utils/RegularUtil.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/RegularUtil.java diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index ffab40e8a..6b824f356 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -38,6 +38,7 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamTaskService; import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent; import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils; +import com.webank.wedatasphere.streamis.jobmanager.utils.RegularUtil; import com.webank.wedatasphere.streamis.jobmanager.vo.BulkUpdateLabel; import com.webank.wedatasphere.streamis.jobmanager.vo.BulkUpdateLabelResponse; import org.apache.commons.collections.CollectionUtils; @@ -137,6 +138,7 @@ public Message updateLabel(HttpServletRequest req, @RequestBody BulkUpdateLabelR throw new JobErrorException(-1, "Have no permission to save StreamJob [" + jobId + "] configuration"); } String label = bulkUpdateLabel.getLabel(); + if (!RegularUtil.matches(label))throw new JobErrorException(-1, "The label content does not meet specifications [" + jobId + "] "); StreamJob job =new StreamJob(); job.setLabel(label); job.setId(jobId); diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/RegularUtil.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/RegularUtil.java new file mode 100644 index 000000000..0148a4b66 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/RegularUtil.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.jobmanager.utils; + +public class RegularUtil { + public static final String LETTER_PATTERN="^.*[a-zA-Z]+.*$";//字母 + public static final String NUMBER_PATTERN="^.*[0-9]+.*$";//数字 + public static final String SPECIAL_CHAR_PATTERN="^.*[/^/$/.//,;:'!@#%&/*/|/?/+/(/)/[/]/{/}]+.*$";//特殊字符 + public static final String PW_LENGTH_PATTERN="^.{0,64}$";//字符长度 + public static final String PATTERN ="^[A-Za-z0-9_,.]{1,64}$" ; //大小写字母数字下划线逗号小数点 + + + public static boolean matches(String input) { + return input.matches(PATTERN); + } + +} From e94e545abf193249cedb54db161dc39460838c7d Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Thu, 25 May 2023 17:55:41 +0800 Subject: [PATCH 166/219] fix import --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 29b90740c..b565fd384 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -22,7 +22,8 @@ import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.JobInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf; - import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.exception.JobErrorException; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobStateInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.jobInfo.EngineConnJobInfo; @@ -32,7 +33,6 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.*; -import com.webank.wedatasphere.streamis.jobmanager.manager.exception.JobErrorException; import com.webank.wedatasphere.streamis.jobmanager.manager.project.service.ProjectPrivilegeService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobInspectService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobService; From a398d4a2cc72834a3e82fda4a90fdba3bef8df83 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Fri, 26 May 2023 11:00:36 +0800 Subject: [PATCH 167/219] add check on-off --- .../jobmanager/utils/HttpClientUtil.java | 16 +++++++++------- .../src/main/resources/linkis.properties | 2 ++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java index b950a343e..be860b051 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java @@ -43,6 +43,7 @@ public class HttpClientUtil { public static final CommonVars LINKIS_GATEWAY_URL = CommonVars.apply("wds.dss.gateway.url", ""); + public static final CommonVars LINKIS_HTTP_IS_REQUEST = CommonVars.apply("wds.linkis.http.is.request", ""); static { Registry socketFactoryRegistry = RegistryBuilder.create() @@ -145,14 +146,15 @@ public static boolean checkSystemAdmin(String token){ String linkisUrl= LINKIS_GATEWAY_URL.getValue()+ "/api/rest_j/v1/jobhistory/governanceStationAdmin"; Gson gson = BDPJettyServerHelper.gson(); boolean flag =false; - try { - String responseStr = get(null, linkisUrl, null, token).getResponseStr(); - LinkisResponseData linkisResponseData = gson.fromJson(responseStr, LinkisResponse.class).getData(); - flag =linkisResponseData.getAdmin(); - } catch (IOException e) { - throw new RuntimeException(e); + if (LINKIS_HTTP_IS_REQUEST.getValue().equals("true")) { + try { + String responseStr = get(null, linkisUrl, null, token).getResponseStr(); + LinkisResponseData linkisResponseData = gson.fromJson(responseStr, LinkisResponse.class).getData(); + flag = linkisResponseData.getAdmin(); + } catch (IOException e) { + logger.error("Failed to request the Linkis system administrator interface"); + } } - return flag; } } diff --git a/streamis-server/src/main/resources/linkis.properties b/streamis-server/src/main/resources/linkis.properties index b09dbbc4a..9c6aaf990 100644 --- a/streamis-server/src/main/resources/linkis.properties +++ b/streamis-server/src/main/resources/linkis.properties @@ -20,6 +20,8 @@ wds.linkis.server.mybatis.datasource.password=pwd1 wds.linkis.gateway.ip= wds.linkis.gateway.port= wds.linkis.gateway.url=http://localhost:9001 +wds.dss.gateway.url=http://localhost:9001 +wds.linkis.http.is.request=ture wds.linkis.mysql.is.encrypt=false ##restful From 07d6dfc539e0c0bfcd42067ea53f624cda1c378a Mon Sep 17 00:00:00 2001 From: Liveipool <123653079@qq.com> Date: Fri, 26 May 2023 18:27:49 +0800 Subject: [PATCH 168/219] fix: bug --- web/src/apps/streamis/module/jobList/index.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 5f04ba3a4..d19b3a55c 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -687,6 +687,8 @@ export default { this.$refs['formValidate'].resetFields(); this.showEditTags = false; this.editTagsLoading = false; + this.pageData.current = 1 + this.getJobList() } catch { this.editTagsLoading = false; } From f5e0f66ea6adb936319a0932d4999310111548c6 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Sun, 28 May 2023 18:04:44 +0800 Subject: [PATCH 169/219] solve sonar smell and bug --- .../SparkStreamJobHeartbeatProducer.scala | 1 - .../launcher/entity/JobConfDefinition.java | 2 +- .../jobmanager/launcher/entity/JobUser.java | 2 +- .../launcher/entity/JobUserRole.java | 2 +- .../state/AbstractLinkisJobStateFetcher.java | 1 - .../job/state/client/StateFileTree.java | 1 - .../launcher/linkis/utils/HttpClientUtil.java | 1 - .../client/LinkisFlinkManagerJobClient.scala | 4 ++-- .../job/client/SparkRestJobClient.scala | 2 +- .../factory/EngineConnJobClientFactory.scala | 4 +++- .../LinkisFlinkManagerClientFactory.scala | 4 +++- .../client/factory/RestJobClientFactory.scala | 7 +------ .../job/jobInfo/EngineConnJobInfo.scala | 1 - .../job/manager/AbstractJobStateManager.scala | 2 +- .../job/manager/FlinkJobLaunchManager.scala | 4 ++-- .../job/manager/FlinkJobStateManager.scala | 4 ++-- .../job/manager/LinkisJobLaunchManager.scala | 4 +--- .../JobLauncherAutoConfiguration.java | 6 +++--- .../service/tools/JobConfValueUtils.scala | 18 ++---------------- .../flink/FlinkStreamisConfigAutowired.java | 4 ---- .../flink/FlinkStreamisConfigDefine.java | 13 +++++++++---- .../flink/FlinkConfigurationLoadTest.java | 2 +- .../config/StreamisLogAppenderConfig.java | 1 - .../sender/AbstractRpcLogSender.java | 13 ++++++------- .../log/collector/sender/SendLogCache.java | 2 +- .../sender/SendLogCacheConsumer.java | 2 +- .../sender/SendLogExceptionStrategy.java | 2 +- .../sender/buf/AbstractSendBuffer.java | 4 ++-- .../sender/http/AbstractHttpLogSender.java | 7 ++----- .../collector/sender/http/HttpClientTool.java | 19 +++++++++---------- .../http/request/AbstractHttpAction.java | 2 +- .../sender/http/request/GetAction.java | 4 ---- .../jobmanager/log/utils/StringUtils.java | 1 + .../log/collector/StreamisRpcLogAppender.java | 9 ++------- .../log/collector/StreamisRpcLogAppender.java | 2 -- .../jobmanager/log/json/JsonTool.java | 1 + .../server/storage/StreamisJobLogStorage.java | 3 --- .../storage/bucket/Log4j2JobLogBucket.java | 2 -- .../storage/bucket/StreamisJobLogBucket.java | 7 ------- .../loadbalancer/RoundRobinLoadBalancer.java | 2 +- .../loadbalancer/SimpleLoadBalancer.java | 2 +- .../jobmanager/log/LogStorageTest.java | 4 ---- .../manager/entity/MetaJsonInfo.java | 1 - .../manager/entity/StreamJobVersionFiles.java | 2 +- .../manager/entity/vo/ExecResultVo.java | 2 +- .../manager/entity/vo/PauseResultVo.java | 2 +- .../exception/FileExceptionManager.java | 3 +++ .../jobmanager/manager/util/CookieUtils.java | 2 ++ .../jobmanager/manager/util/IoUtils.java | 2 ++ .../jobmanager/manager/util/ZipHelper.java | 5 ++++- .../manager/JobManagerAutoConfiguration.java | 1 - .../jobmanager/manager/alert/AlertConf.java | 8 ++++---- .../manager/alert/DefaultAlerter.java | 1 - .../manager/scheduler/StreamisScheduler.java | 1 + .../StreamisSchedulerExecutorManager.java | 6 +++--- .../scheduler/TenancyConsumerManager.java | 8 ++++---- .../AbstractStreamisSchedulerEvent.java | 3 +-- .../service/DefaultStreamJobService.scala | 7 +------ .../service/DefaultStreamTaskService.scala | 2 +- .../StreamisJobContentTransform.scala | 1 - .../AbstractStreamisTransformJobBuilder.scala | 2 -- .../restful/api/JobConfExtRestfulApi.java | 3 +-- .../restful/api/JobConfRestfulApi.java | 1 - .../restful/api/ProjectRestfulApi.java | 2 +- 64 files changed, 94 insertions(+), 149 deletions(-) delete mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java diff --git a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/entrypoint/producer/SparkStreamJobHeartbeatProducer.scala b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/entrypoint/producer/SparkStreamJobHeartbeatProducer.scala index 85f21cdda..dfadf61a6 100644 --- a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/entrypoint/producer/SparkStreamJobHeartbeatProducer.scala +++ b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/entrypoint/producer/SparkStreamJobHeartbeatProducer.scala @@ -9,7 +9,6 @@ class SparkStreamJobHeartbeatProducer extends StreamJobHeartbeatProducer { var message: JobHeartbeatMessage = new JobHeartbeatMessage message.setStreamJobConfig(streamJobConfig) message.setEngineType("spark") - //todo get engineVersion from env message } } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobConfDefinition.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobConfDefinition.java index 3f6e2ce4e..83409e3cf 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobConfDefinition.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobConfDefinition.java @@ -1,6 +1,6 @@ /* * Copyright 2021 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java index acb9cfafa..8a5cde7d3 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java @@ -1,6 +1,6 @@ /* * Copyright 2021 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java index 616bdd04b..c78ad3f73 100755 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java @@ -1,6 +1,6 @@ /* * Copyright 2021 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/AbstractLinkisJobStateFetcher.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/AbstractLinkisJobStateFetcher.java index f2f650156..da97bdf12 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/AbstractLinkisJobStateFetcher.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/AbstractLinkisJobStateFetcher.java @@ -21,7 +21,6 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobStateFetcher; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.exception.FlinkJobStateFetchException; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.exception.StreamisJobLaunchException; -import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.client.LinkisFlinkManagerJobClient; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.client.StateFileTree; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.client.LinkisJobStateGetAction; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.client.LinkisJobStateResult; diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/client/StateFileTree.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/client/StateFileTree.java index 89f236695..e193aeb66 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/client/StateFileTree.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/state/client/StateFileTree.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.client; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java index 4d0330c8c..c769c372f 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.utils; -import org.apache.commons.lang3.StringUtils; import org.apache.http.Consts; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala index 216cab71e..9a7667c0c 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala @@ -45,7 +45,7 @@ class LinkisFlinkManagerJobClient(onceJob: OnceJob, jobInfo: JobInfo, stateManag case JobClientType.DETACH => true case JobClientType.DETACH_STANDALONE => - // TODO check + true case _ => throw new FlinkJobParamErrorException(s"Job with manager mode : ${jobInfo.getClientType} cannot be submited.", null) @@ -55,7 +55,7 @@ class LinkisFlinkManagerJobClient(onceJob: OnceJob, jobInfo: JobInfo, stateManag override def getJobInfo(refresh: Boolean): JobInfo = { onceJob match { case simpleOnceJob: SimpleOnceJob => - // TODO check + if (refresh && isDetachJob(jobInfo)) { jobInfo match { case engineConnJobInfo: EngineConnJobInfo => diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/SparkRestJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/SparkRestJobClient.scala index d162934ac..06d449cb5 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/SparkRestJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/SparkRestJobClient.scala @@ -13,6 +13,6 @@ class SparkRestJobClient(onceJob: OnceJob, jobInfo: JobInfo, stateManager: JobSt * Stop */ override def stop(): Unit = { - + //nothing } } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/EngineConnJobClientFactory.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/EngineConnJobClientFactory.scala index 11571d496..ceb5416d6 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/EngineConnJobClientFactory.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/EngineConnJobClientFactory.scala @@ -29,5 +29,7 @@ class EngineConnJobClientFactory extends JobClientFactory { /** * Init the factory */ - override def init(): Unit = {} + override def init(): Unit = { + //init + } } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/LinkisFlinkManagerClientFactory.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/LinkisFlinkManagerClientFactory.scala index 00a5a05ae..5aecf3396 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/LinkisFlinkManagerClientFactory.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/LinkisFlinkManagerClientFactory.scala @@ -17,5 +17,7 @@ class LinkisFlinkManagerClientFactory extends JobClientFactory { /** * Init the factory */ - override def init(): Unit = {} + override def init(): Unit = { + //init + } } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/RestJobClientFactory.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/RestJobClientFactory.scala index 359a6c325..90d817ddb 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/RestJobClientFactory.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/factory/RestJobClientFactory.scala @@ -26,12 +26,7 @@ class RestJobClientFactory extends JobClientFactory { */ override def createJobClient(onceJob: OnceJob, jobInfo: JobInfo, jobStateManager: JobStateManager): JobClient[LinkisJobInfo] = { - - //todo create flink spark client - - - - + //create flink spark client null } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/jobInfo/EngineConnJobInfo.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/jobInfo/EngineConnJobInfo.scala index 3605b904b..1899a1430 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/jobInfo/EngineConnJobInfo.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/jobInfo/EngineConnJobInfo.scala @@ -17,7 +17,6 @@ class EngineConnJobInfo extends YarnJobInfo with LinkisJobInfo { private var jobStates: Array[JobStateInfo] = _ private var engineType: String = "flink" private var engineVersion: String = "1.12.2" - // TODO check private var clientType: String = JobClientType.ATTACH.toString private var savepoint: String = _ diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/AbstractJobStateManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/AbstractJobStateManager.scala index 767b9e58e..267d3bf73 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/AbstractJobStateManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/AbstractJobStateManager.scala @@ -107,7 +107,7 @@ abstract class AbstractJobStateManager extends JobStateManager { path = path.replaceAll("/+", "/") // Replace "." to "/" path = path.replaceAll("\\.", "/") - if (path.endsWith("/") && !(path == "/") && !WINDOWS_ROOT_DIR_REGEX.pattern.matcher(path).matches()) path = path.substring(0, path.length - "/".length) + if (path.endsWith("/") && (path != "/") && !WINDOWS_ROOT_DIR_REGEX.pattern.matcher(path).matches()) path = path.substring(0, path.length - "/".length) path } def getJobStateRootPath[T <: JobState](clazz: Class[_], schema: String): String diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala index 06410c0a3..28d245c22 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala @@ -72,7 +72,7 @@ trait FlinkJobLaunchManager extends LinkisJobLaunchManager with Logging { case engineConnType: String => if(!engineConnType.toLowerCase.startsWith(FlinkJobLaunchManager.FLINK_ENGINE_CONN_TYPE)) throw new FlinkJobLaunchErrorException(30401, s"Only ${FlinkJobLaunchManager.FLINK_ENGINE_CONN_TYPE} job is supported to be launched to Linkis, but $engineConnType is found.(不识别的引擎类型)", null) - //todo add flink and spark + case _ => throw new FlinkJobLaunchErrorException(30401, s"Not exists ${LabelKeyUtils.ENGINE_TYPE_LABEL_KEY}(缺少引擎标签), StreamisJob cannot be submitted to Linkis successfully.", null) } Utils.tryCatch { @@ -115,7 +115,7 @@ trait FlinkJobLaunchManager extends LinkisJobLaunchManager with Logging { connect(id, createJobInfo(jobInfo)) } - // todo + override def connect(id: String, jobInfo: LinkisJobInfo): JobClient[LinkisJobInfo] = { AbstractJobClientFactory.getJobManager().createJobClient(createSubmittedOnceJob(id, jobInfo), jobInfo, getJobStateManager) } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobStateManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobStateManager.scala index 76600815e..6eb03d463 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobStateManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobStateManager.scala @@ -30,7 +30,7 @@ class FlinkJobStateManager extends AbstractJobStateManager with Logging{ */ override def init(): Unit = { info("Register the loader for JobState fetcher") - // TODO register the fetcher + registerJobStateFetcher(classOf[FlinkCheckpoint], () => new FlinkCheckpointJobStateFetcher(classOf[FlinkCheckpoint], this)) registerJobStateFetcher(classOf[FlinkSavepoint], () => new FlinkSavepointJobStateFetcher(classOf[FlinkSavepoint], this)) } @@ -51,6 +51,6 @@ object FlinkJobStateManager{ URL.setURLStreamHandlerFactory(new LinkisURLStreamHandlerFactory(JobLauncherConfiguration.FLINK_STATE_SUPPORT_SCHEMES.getValue.split(","): _*)) def main(args: Array[String]): Unit = { - + //nothing } } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/LinkisJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/LinkisJobLaunchManager.scala index d1b8869be..21cef7ec0 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/LinkisJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/LinkisJobLaunchManager.scala @@ -42,9 +42,7 @@ trait LinkisJobLaunchManager extends JobLaunchManager[LinkisJobInfo] with Loggin } } - // Set labels to launchJob -// job.getLabels.put("engineType", "") //todo -// job.getLabels.put("linkisVersion", linkisVersion) + if (StringUtils.isNotBlank(linkisVersion)){ val versionSplitter: Array[String] = linkisVersion.split("\\.") diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncherAutoConfiguration.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncherAutoConfiguration.java index 2c6de1899..dc96e5c4b 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncherAutoConfiguration.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncherAutoConfiguration.java @@ -63,9 +63,9 @@ public JobLaunchManager defaultJobLaunchManager(){ } }); // Add shutdown hook to destroy the launch manager - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - JobLaunchManager$.MODULE$.getJobManagers().forEach(JobLaunchManager::destroy); - })); + Runtime.getRuntime().addShutdownHook(new Thread(() -> + JobLaunchManager$.MODULE$.getJobManagers().forEach(JobLaunchManager::destroy) + )); // Use the flink job launch manager as default JobLaunchManager defaultManager = JobLaunchManager$.MODULE$.getJobManager(DEFAULT_JOB_LAUNCH_MANGER); if (Objects.isNull(defaultManager)){ diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/tools/JobConfValueUtils.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/tools/JobConfValueUtils.scala index e8b5d71bb..2640d4b3f 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/tools/JobConfValueUtils.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/tools/JobConfValueUtils.scala @@ -23,7 +23,7 @@ import org.apache.commons.lang.StringUtils import scala.collection.JavaConverters._ import java.util /** - * TODO dive into JobConfValueSerializer and JobConfValueDeserializer + * dive into JobConfValueSerializer and JobConfValueDeserializer */ class JobConfValueUtils { @@ -180,19 +180,5 @@ object JobConfValueUtils{ } result } -// def main(args: Array[String]): Unit = { -// val definitions: util.List[JobConfDefinition] = new util.ArrayList[JobConfDefinition]() -// val configValues: util.List[JobConfValue] = new util.ArrayList[JobConfValue]() -// definitions.add(new JobConfDefinition(0, "wds.linkis.flink.resource", "None", null, 0)) -// definitions.add(new JobConfDefinition(1, "wds.linkis.flink.custom", "None", null, 0)) -// definitions.add(new JobConfDefinition(2, "wds.linkis.flink.taskmanager.num", "NUMBER", 0, 1)) -// definitions.add(new JobConfDefinition(3, "wds.linkis.flink.jobmanager.memeory", "NUMBER", 0, 1)) -// configValues.add(new JobConfValue("wds.linkis.flink.taskmanager.num", "1", 2)) -// configValues.add(new JobConfValue("env.java.opts", "-DHADOOP_USER_NAME=hadoop", 1)) -// configValues.add(new JobConfValue("security.kerberos.login.principal", "hadoop@WEBANK.com", 1)) -// configValues.add(new JobConfValue("wds.linkis.flink.jobmanager.memeory", "1024", 3)) -// val result = serialize(configValues, definitions) -// println(DWSHttpClient.jacksonJson.writeValueAsString(result)) -// println(DWSHttpClient.jacksonJson.writeValueAsString(deserialize(result, definitions))) -// } + } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index a5459021c..0d2b19b3c 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -112,10 +112,6 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui * @return configuration */ private synchronized Configuration loadConfiguration(){ -// String configDir = System.getenv("FLINK_CONF_DIR"); -// if (null == configDir){ -// configDir = "."; -// } String configDir = "."; Properties properties = System.getProperties(); Enumeration enumeration = properties.propertyNames(); diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java index 5f23ae681..37da9329b 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java @@ -2,7 +2,6 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; -import scala.Int; import java.util.List; @@ -11,6 +10,11 @@ */ public class FlinkStreamisConfigDefine { + private FlinkStreamisConfigDefine(){} + + + private static String Error ="ERROR"; + /** * Gateway address of log module for streamis */ @@ -122,13 +126,14 @@ public class FlinkStreamisConfigDefine { * Level value of LevelMatch filter strategy */ public static final ConfigOption LOG_FILTER_LEVEL_MATCH = ConfigOptions.key("stream.log.filter.level-match.level") - .stringType().defaultValue("ERROR").withDescription("Level value of LevelMatch filter strategy"); + .stringType().defaultValue(Error).withDescription("Level value of LevelMatch filter strategy"); + /** * Level value of ThresholdMatch filter strategy */ public static final ConfigOption LOG_FILTER_THRESHOLD_MATCH = ConfigOptions.key("stream.log.filter.threshold.level") - .stringType().defaultValue("ERROR").withDescription("Level value of ThresholdMatch filter strategy"); + .stringType().defaultValue(Error).withDescription("Level value of ThresholdMatch filter strategy"); /** * Regex value of RegexMatch filter strategy */ @@ -139,7 +144,7 @@ public class FlinkStreamisConfigDefine { * Accept keywords of Keyword filter strategy */ public static final ConfigOption LOG_FILTER_KEYWORDS = ConfigOptions.key("stream.log.filter.keywords") - .stringType().defaultValue("ERROR").withDescription("Accept keywords of Keyword filter strategy"); + .stringType().defaultValue(Error).withDescription("Accept keywords of Keyword filter strategy"); /** * Exclude keywords of Keyword filter strategy diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java index 79ad46014..bfd97940d 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java @@ -11,7 +11,7 @@ import java.util.Properties; public class FlinkConfigurationLoadTest { - private static final Logger LOG = LoggerFactory.getLogger(FlinkConfigurationLoadTest.class); + @Test public void loadConfiguration() { String configDir = Objects.requireNonNull(FlinkConfigurationLoadTest.class.getResource("/")).getFile(); diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 76fbd0c91..f874c8d8b 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.Optional; /** diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index 3658a335e..d8e98297e 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -8,6 +8,7 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import java.io.IOException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; @@ -53,7 +54,8 @@ public abstract class AbstractRpcLogSender implements R */ protected ExceptionListener exceptionListener; - public AbstractRpcLogSender(RpcLogSenderConfig rpcSenderConfig){ + + protected AbstractRpcLogSender(RpcLogSenderConfig rpcSenderConfig){ this.rpcSenderConfig = rpcSenderConfig; SendLogCacheConfig cacheConfig = rpcSenderConfig.getCacheConfig(); this.cacheSize = cacheConfig.getSize(); @@ -111,7 +113,7 @@ public void close() { * @param aggregatedEntity agg entity * @param rpcSenderConfig rpc sender config */ - protected abstract void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) throws Exception; + protected abstract void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) throws IOException; /** * Send log exception strategy @@ -190,7 +192,7 @@ public Thread newThread(Runnable r) { public boolean startCacheConsumer(){ if (consumers >= maxCacheConsume) { -// throw new IllegalStateException("Over the limit number of cache consumers: [" + maxCacheConsume + "]"); + return false; } this.ctxLock.lock(); @@ -332,14 +334,11 @@ public void cacheLog(T logElement) throws InterruptedException { } try{ flowControl(); - if (discard && control.decrementAndGet() <= 0){ - if (logElement.mark() < 2){ + if (discard && control.decrementAndGet() <= 0 && logElement.mark() < 2){ discardCount++; return; - } } while (count == items.length){ -// System.out.println("The queue is full, maybe lost the data"); long ws = System.currentTimeMillis(); notFull.await(); cacheWaitTime.addAndGet(System.currentTimeMillis() - ws); diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java index 200c573d9..56b627905 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java @@ -16,5 +16,5 @@ public interface SendLogCache extends LogCache { * @param maxElements max element size * @return count */ - int drainLogsTo(SendBuffer sendBuffer, int maxElements);; + int drainLogsTo(SendBuffer sendBuffer, int maxElements); } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index 0600e8d6a..045155eeb 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -37,7 +37,7 @@ public abstract class SendLogCacheConsumer implements Runn */ private Future future; - public SendLogCacheConsumer(String id, SendLogCache cache, + protected SendLogCacheConsumer(String id, SendLogCache cache, SendBuffer sendBuffer, RpcLogSenderConfig rpcSenderConfig){ this.id = id; diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java index d33b7d2e4..a547fb534 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java @@ -13,7 +13,7 @@ public abstract class SendLogExceptionStrategy { protected final RpcLogSender sender; - public SendLogExceptionStrategy(RpcLogSender sender){ + protected SendLogExceptionStrategy(RpcLogSender sender){ this.sender = sender; } /** diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java index 1b42ad957..c0feee3b9 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java @@ -23,12 +23,12 @@ protected enum Flag{ protected int capacity; - public AbstractSendBuffer(int capacity){ + protected AbstractSendBuffer(int capacity){ this.capacity = capacity; limit(this.capacity); } - public AbstractSendBuffer(){ + protected AbstractSendBuffer(){ this(Integer.MAX_VALUE); } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index c693d0152..07ec71601 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -44,7 +44,7 @@ public abstract class AbstractHttpLogSender extends Abs */ private final AtomicLong serverRecoveryTimePoint = new AtomicLong(-1L); - public AbstractHttpLogSender(RpcLogSenderConfig rpcSenderConfig) { + protected AbstractHttpLogSender(RpcLogSenderConfig rpcSenderConfig) { super(rpcSenderConfig); this.globalHttpClient = HttpClientTool.createHttpClient(rpcSenderConfig); this.sendRetryStrategy = new SendLogExceptionStrategy(this) { @@ -71,17 +71,14 @@ public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBu break; } } - if (!shouldRetry && e instanceof HttpResponseException){ - if (((HttpResponseException) e).getStatusCode() < 500){ + if (!shouldRetry && e instanceof HttpResponseException && ((HttpResponseException) e).getStatusCode() < 500){ shouldRetry = true; - } } } if (shouldRetry && !sender.getOrCreateLogCache().isCacheable()){ // Means that the cache is full // Set the position of buffer to 0 sendBuffer.rewind(); - // Compact the buffer and transient to write mode; sendBuffer.compact( element -> element.mark() > 1); shouldRetry = false; } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java index 12f3f7dab..7f62fc6fe 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java @@ -18,6 +18,8 @@ */ public class HttpClientTool { + private HttpClientTool() {} + /** * Connect timeout */ @@ -57,16 +59,13 @@ public static HttpClient createHttpClient(RpcLogSenderConfig rpcSenderConfig){ clientBuilder.setDefaultRequestConfig(requestConfig).setDefaultHeaders(defaultHeaders) .useSystemProperties().setMaxConnTotal(maxConn).setMaxConnPerRoute(maxConn); CloseableHttpClient httpClient = clientBuilder.build(); - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - try { - httpClient.close(); - } catch (IOException e) { - // Ignore - } - } - })); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + httpClient.close(); + } catch (IOException e) { + // Ignore + } + })); return httpClient; } } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java index 143f72b8e..b35a17d35 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java @@ -20,7 +20,7 @@ public abstract class AbstractHttpAction implements H protected String user; - public AbstractHttpAction(String uri){ + protected AbstractHttpAction(String uri){ this.uri = uri; } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java deleted file mode 100644 index f5a8a5fef..000000000 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; - -public class GetAction { -} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/utils/StringUtils.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/utils/StringUtils.java index 4b1660e65..6ccc26e34 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/utils/StringUtils.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/utils/StringUtils.java @@ -5,6 +5,7 @@ */ public class StringUtils { + private StringUtils(){} /** * Convert string to array * @param input string diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index a82f44cbb..f0597f47e 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -20,12 +20,8 @@ import org.apache.logging.log4j.core.layout.PatternLayout; import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.ServiceLoader; +import java.util.*; import java.util.function.BiFunction; -import java.util.function.Function; /** * Streamis rpc log appender @@ -80,9 +76,8 @@ protected StreamisRpcLogAppender(String name, Filter filter, @Override public void append(LogEvent event) { - String content = new String(getLayout().toByteArray(event)); + String content = Arrays.toString(getLayout().toByteArray(event)); if (messageFilterFunction.apply(event.getLoggerName(), content)) { - // Transform to stream log event; StreamisLogEvent logEvent = new StreamisLogEvent(content, event.getTimeMillis()); try { this.logCache.cacheLog(logEvent); diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index ab91077b4..50760c7a5 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -57,8 +57,6 @@ public class StreamisRpcLogAppender extends AppenderSkeleton { protected void append(LoggingEvent loggingEvent) { String content = super.getLayout().format(loggingEvent); if (messageFilterFunction.apply(loggingEvent.getLoggerName(), content)) { - // Transform to stream log event; - // System.currentTimeMills() -> loggingEvent.getTimeStamp() StreamisLogEvent logEvent = new StreamisLogEvent(content, loggingEvent.getTimeStamp()); if (Objects.nonNull(logCache)) { try { diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java index 0822e820d..9d17492dd 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java @@ -5,6 +5,7 @@ public class JsonTool { static final char[] HEX_DIGITS = new char[] {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; + private JsonTool(){} /** * Avoid the special char * @param input input string diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index ccadd721d..c04045ee9 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -8,14 +8,11 @@ import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketState; import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.context.*; import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.loadbalancer.JobLogStorageLoadBalancer; -import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.loadbalancer.RoundRobinLoadBalancer; import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.utils.MemUtils; import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index dd564c451..0f2641185 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -17,7 +17,6 @@ import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.LoggerConfig; import org.apache.logging.log4j.core.layout.PatternLayout; -import org.checkerframework.checker.units.qual.A; import org.slf4j.LoggerFactory; import java.io.File; @@ -330,7 +329,6 @@ private RolloverStrategy createRolloverStrategy(Configuration configuration, * @return file name with absolute path */ private String resolveFileName(String bucketRootPath, String bucketName){ - // {projectName}.{jobName} String fileName = FilenameUtils.normalize(bucketName); String basePath = bucketRootPath; if (!basePath.endsWith("/")){ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java deleted file mode 100644 index ba9c002d6..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; - -/** - * Use the appender and strategy of log4j (version 1.x) to implement the bucket - */ -public class StreamisJobLogBucket { -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/RoundRobinLoadBalancer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/RoundRobinLoadBalancer.java index 8d77d5fd1..ef12087cb 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/RoundRobinLoadBalancer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/RoundRobinLoadBalancer.java @@ -43,7 +43,7 @@ public int priority() { @Override public void init() { - + //init } @Override diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/SimpleLoadBalancer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/SimpleLoadBalancer.java index e11e9fa9c..42b20133e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/SimpleLoadBalancer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/loadbalancer/SimpleLoadBalancer.java @@ -33,7 +33,7 @@ public void onContextEvent(ContextEvent event) { @Override public void init() { - + //init } @Override diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java deleted file mode 100644 index 6ced214f2..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log; - -public class LogStorageTest { -} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/MetaJsonInfo.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/MetaJsonInfo.java index 41032c6c8..7c4d7b743 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/MetaJsonInfo.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/MetaJsonInfo.java @@ -16,7 +16,6 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.entity; import javax.validation.constraints.NotBlank; -import java.util.HashMap; import java.util.Map; /** diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersionFiles.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersionFiles.java index 3f0f83187..cd52691a7 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersionFiles.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersionFiles.java @@ -1,6 +1,6 @@ /* * Copyright 2021 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/ExecResultVo.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/ExecResultVo.java index 1e101a8a4..375594144 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/ExecResultVo.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/ExecResultVo.java @@ -1,6 +1,6 @@ /* * Copyright 2021 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PauseResultVo.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PauseResultVo.java index ac41b5d0b..873fee724 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PauseResultVo.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PauseResultVo.java @@ -1,6 +1,6 @@ /* * Copyright 2021 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/exception/FileExceptionManager.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/exception/FileExceptionManager.java index 22153bb25..e37b0e135 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/exception/FileExceptionManager.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/exception/FileExceptionManager.java @@ -20,6 +20,9 @@ import java.util.Map; public class FileExceptionManager { + + private FileExceptionManager(){} + //30600-30700 private static Map desc = new HashMap(32); static { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/CookieUtils.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/CookieUtils.java index 9cd05e9e2..f8e9d94af 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/CookieUtils.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/CookieUtils.java @@ -22,6 +22,8 @@ public class CookieUtils { private static final String COOKIE_WORKSPACE_ID = "workspaceId"; + private CookieUtils(){} + static String getCookieValue(HttpServletRequest request, String name) { Cookie c = getCookie(request, name); return c == null ? null : c.getValue(); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/IoUtils.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/IoUtils.java index 9324f0f50..d79d76271 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/IoUtils.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/IoUtils.java @@ -32,6 +32,8 @@ public class IoUtils { private static final String dateFormat_time = "HHmmss"; private static final String IOUrl = CommonVars.apply("wds.streamis.zip.dir", "/tmp").getValue(); + private IoUtils(){} + public static String generateIOPath(String userName, String projectName, String subDir) { String baseIOUrl = IOUrl; String file = subDir.substring(0,subDir.lastIndexOf(".")); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ZipHelper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ZipHelper.java index d037d2c51..e17ac5a75 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ZipHelper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ZipHelper.java @@ -34,7 +34,10 @@ public class ZipHelper { private static final String RECURSIVE = "-r"; private static final String ZIP_TYPE = ".zip"; - public static String unzip(String dirPath)throws Exception { //"D:\\tmp\\streamis\\20210922\\johnnwang\\ab_175950\\ab.zip" + private ZipHelper(){} + + public static String unzip(String dirPath)throws Exception { + //"D:\\tmp\\streamis\\20210922\\johnnwang\\ab_175950\\ab.zip" File file = new File(dirPath); if(!file.exists()){ logger.error("{} does not exist, can not unzip", dirPath); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/JobManagerAutoConfiguration.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/JobManagerAutoConfiguration.java index 98f0267c9..391490cd0 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/JobManagerAutoConfiguration.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/JobManagerAutoConfiguration.java @@ -19,7 +19,6 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.StreamisScheduler; import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.StreamisSchedulerExecutorManager; import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.TenancyConsumerManager; -import org.apache.linkis.scheduler.Scheduler; import org.apache.linkis.scheduler.executer.ExecutorManager; import org.apache.linkis.scheduler.queue.ConsumerManager; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/AlertConf.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/AlertConf.java index e0361e7c4..b5771c60c 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/AlertConf.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/AlertConf.java @@ -21,13 +21,13 @@ import org.apache.linkis.common.conf.CommonVars; -public interface AlertConf { +public class AlertConf { - CommonVars ALERT_IP = CommonVars.apply("wds.streamis.alert.streamis.ip", "127.0.0.1"); + public static CommonVars ALERT_IP = CommonVars.apply("wds.streamis.alert.streamis.ip", "127.0.0.1"); - CommonVars ALERT_SUB_SYS_ID = CommonVars.apply("wds.streamis.alert.streamis.systemid", "7495"); + public static CommonVars ALERT_SUB_SYS_ID = CommonVars.apply("wds.streamis.alert.streamis.systemid", "7495"); - Gson COMMON_GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + public static Gson COMMON_GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/DefaultAlerter.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/DefaultAlerter.java index 85f820417..516ff9ee7 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/DefaultAlerter.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/alert/DefaultAlerter.java @@ -17,7 +17,6 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.dao.StreamAlertMapper; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamAlertRecord; -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisScheduler.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisScheduler.java index feadc2bab..5de042b37 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisScheduler.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisScheduler.java @@ -44,6 +44,7 @@ public class StreamisScheduler extends AbstractScheduler implements FutureSchedu public static class Constraints{ + private Constraints (){} private static final CommonVars TENANCY_PATTERN = CommonVars.apply("wds.streamis.job.scheduler.consumer.tenancies", "hadoop"); private static final CommonVars GROUP_INIT_CAPACITY = CommonVars.apply("wds.streamis.job.scheduler.group.min.capacity", 1000); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisSchedulerExecutorManager.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisSchedulerExecutorManager.java index 602d0d82f..dbd1eda5b 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisSchedulerExecutorManager.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/StreamisSchedulerExecutorManager.java @@ -57,12 +57,12 @@ public Executor[] getByGroup(String groupName) { @Override public void delete(Executor executor) { - + //nothing } @Override public void shutdown() { - + //nothing } private Executor getOrCreateExecutor(){ @@ -114,7 +114,7 @@ public ExecutorInfo getExecutorInfo() { @Override public void close() throws IOException { - + //nothing } } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/TenancyConsumerManager.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/TenancyConsumerManager.java index 896d1d465..dd0fea507 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/TenancyConsumerManager.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/TenancyConsumerManager.java @@ -134,11 +134,11 @@ protected ExecutorService getOrCreateExecutorService(String groupName){ String tenancy = tenancyGroupFactory.getTenancyByGroupName(groupName); groupFactory.getGroup(groupName); if (StringUtils.isNotBlank(tenancy)){ - return tenancyExecutorServices.computeIfAbsent(tenancy, tenancyName -> { + return tenancyExecutorServices.computeIfAbsent(tenancy, tenancyName -> // Use the default value of max running jobs - return Utils.newCachedThreadPool(tenancyGroupFactory.getDefaultMaxRunningJobs() + 1, - TenancyGroupFactory.GROUP_NAME_PREFIX + tenancy + "-Executor-", true); - }); + Utils.newCachedThreadPool(tenancyGroupFactory.getDefaultMaxRunningJobs() + 1, + TenancyGroupFactory.GROUP_NAME_PREFIX + tenancy + "-Executor-", true) + ); } } return getOrCreateExecutorService(); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/AbstractStreamisSchedulerEvent.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/AbstractStreamisSchedulerEvent.java index 383dba8b6..38dfe6a97 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/AbstractStreamisSchedulerEvent.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/AbstractStreamisSchedulerEvent.java @@ -15,7 +15,6 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events; -import com.ctc.wstx.util.StringUtil; import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.StreamisSchedulerEvent; import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.exception.StreamisScheduleException; import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.exception.StreamisScheduleRetryException; @@ -73,7 +72,7 @@ public abstract class AbstractStreamisSchedulerEvent extends Job implements Stre */ protected Map resultSet = new HashMap<>(); - public AbstractStreamisSchedulerEvent(){ + protected AbstractStreamisSchedulerEvent(){ setJobListener(new JobListener() { @Override public void onJobInited(Job job) { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index 4e25c4bae..16b754ad1 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -295,16 +295,11 @@ class DefaultStreamJobService extends StreamJobService with Logging { Option(streamJobMapper.queryAndLockJobInCondition(projectName, jobName)) match { case Some(streamJob) => var updateVersion = true - // Use the project privilege at restful api - // if (streamJob.getCreateBy != userName) - // throw new JobCreateErrorException(30030, s"You have no permission to update StreamJob-$jobName.") - // Get the latest task directly val task = streamTaskMapper.getLatestByJobId(streamJob.getId) if (task != null && !JobConf.isCompleted(task.getStatus)) { logger.warn(s"StreamJob-$jobName is in status ${task.getStatus}, your deployment will not update the version in job") updateVersion = false - // throw new JobCreateErrorException(30030, s"StreamJob-$jobName is in status ${tasks.get(0).getStatus}, you cannot upload the zip.") - } + } JobDeployValidateResult(streamJob, updateVersion) case _ => JobDeployValidateResult(null, updateVersion = true) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index b8a059f21..9c67313b0 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -201,7 +201,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ // Launch entrance launch(newTaskId.asInstanceOf[Long], execUser, jobState) } else { - // TODO cannot find the new task id + // cannot find the new task id } null } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/StreamisJobContentTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/StreamisJobContentTransform.scala index ec38fc6ab..2c3f58856 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/StreamisJobContentTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/StreamisJobContentTransform.scala @@ -25,7 +25,6 @@ trait StreamisJobContentTransform extends Transform { override def transform(streamisTransformJob: StreamisTransformJob, job: LaunchJob): LaunchJob = { val jobContent = transformJobContent(streamisTransformJob.getStreamisTransformJobContent) if(jobContent != null) { -// jobContent.put("runType", streamisTransformJob.getStreamisJobEngineConn.getRunType.toString) jobContent.put("runType", streamisTransformJob.getStreamisJobConnect.getRunType.toString) LaunchJob.builder().setLaunchJob(job).setJobContent(jobContent).build() } else job diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index e4c750c22..8ac3baef8 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -50,7 +50,6 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB jobConfig.put(JobConfKeyConstants.GROUP_INTERNAL.getValue, internalGroup) internalLogConfig(internalGroup) transformJob.setConfigMap(jobConfig) -// transformJob.setConfig(configurationService.getFullTree(streamJob.getId)) val streamJobVersions = streamJobMapper.getJobVersions(streamJob.getId) // 无需判断streamJobVersions是否非空，因为TaskService已经判断了 transformJob.setStreamJobVersion(streamJobVersions.get(0)) @@ -78,7 +77,6 @@ abstract class AbstractDefaultStreamisTransformJobBuilder extends AbstractStream case transformJob: StreamisTransformJobImpl => val engineConn = new StreamisJobEngineConnImpl engineConn.setEngineConnType("flink-" + flinkVersion) -// engineConn.setRunType(getRunType(transformJob)) transformJob.setStreamisJobEngineConn(engineConn) val streamisJobConnect = new StreamisJobConnectImpl streamisJobConnect.setRunType(getRunType(transformJob)) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java index 09e31a86c..c9bdaa857 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java @@ -19,7 +19,6 @@ import com.webank.wedatasphere.streamis.jobmanager.service.UserService; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.Message; -import org.apache.linkis.server.security.SecurityFilter; import org.apache.linkis.server.utils.ModuleUserUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +44,7 @@ public Message getWorkspaceUsers(HttpServletRequest req) { List userList = new ArrayList<>(); String workspaceId = CookieUtils.getCookieWorkspaceId(req); if (StringUtils.isNotBlank(workspaceId)) { - String userName = ModuleUserUtils.getOperationUser(req); + userList.addAll(userService.workspaceUserQuery(req, workspaceId)); } else { LOG.warn("Cannot find the workspaceID from DSS，perhaps the cookie value has been lost in request from: {}", req.getLocalAddr()); diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java index 3230255ca..369d9d72f 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java @@ -27,7 +27,6 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobService; import org.apache.linkis.httpclient.dws.DWSHttpClient; import org.apache.linkis.server.Message; -import org.apache.linkis.server.security.SecurityFilter; import org.apache.linkis.server.utils.ModuleUserUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java index 32c7c4164..6a0660ffb 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java @@ -20,7 +20,7 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobService; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.Message; -import org.apache.linkis.server.security.SecurityFilter; + import org.apache.linkis.server.utils.ModuleUserUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; From 295735525156c190bff65b5fe3e64208a95db6d5 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Mon, 29 May 2023 10:50:30 +0800 Subject: [PATCH 170/219] solve sonar smell and bug --- .../StreamJobHeartbeatListener.java | 2 +- .../sender/AbstractRpcLogSender.java | 2 +- .../log/collector/StreamisRpcLogAppender.java | 1 + .../collector/StreamisLogAppenderTest.java | 1 + .../server/storage/StreamisJobLogStorage.java | 1 - .../storage/bucket/Log4j2JobLogBucket.java | 8 ++--- .../restful/api/JobConfExtRestfulApi.java | 1 - .../jobmanager/restful/api/JobRestfulApi.java | 5 +--- .../restful/api/ProjectRestfulApi.java | 1 - .../service/impl/UserServiceImpl.java | 2 +- .../rpc/StreamisJobManagerReceiver.scala | 2 +- .../restful/api/ProjectManagerRestfulApi.java | 3 +- .../projectmanager/utils/DateUtils.java | 7 +++-- .../constant/ProjectUserPrivilegeEnum.java | 6 ---- .../exception/ValidExceptionHander.java | 2 +- .../StreamisProjectPrivilegeRestfulApi.java | 1 - .../restful/StreamisProjectRestfulApi.java | 1 - .../utils/StreamisProjectPrivilegeUtils.java | 5 ++++ .../utils/StreamisProjectRestfulUtils.java | 2 +- .../StreamisProjectServerReceiver.scala | 29 ++----------------- 20 files changed, 24 insertions(+), 58 deletions(-) diff --git a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/entrypoint/StreamJobHeartbeatListener.java b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/entrypoint/StreamJobHeartbeatListener.java index 42e3d1f42..3f7a62962 100644 --- a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/entrypoint/StreamJobHeartbeatListener.java +++ b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/entrypoint/StreamJobHeartbeatListener.java @@ -12,7 +12,7 @@ */ public class StreamJobHeartbeatListener { - volatile Map serviceAndPortMap; + Map serviceAndPortMap; /** * Need to launch diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index d8e98297e..3ad7d9b66 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -46,7 +46,7 @@ public abstract class AbstractRpcLogSender implements R /** * Rpc log context */ - private volatile RpcLogContext rpcLogContext; + private RpcLogContext rpcLogContext; protected boolean isTerminated = false; /** diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index f0597f47e..4f1ea8f93 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -83,6 +83,7 @@ public void append(LogEvent event) { this.logCache.cacheLog(logEvent); } catch (InterruptedException e) { LOGGER.error("StreamisRpcLogAppender: {} interrupted when cache the log into the RPC sender, message: {}", this.getName(), e.getMessage()); + Thread.currentThread().interrupt(); } } } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java index 0bc49c139..fe76139a4 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -19,6 +19,7 @@ public void appenderLog() throws InterruptedException { Thread.sleep(sleep); } catch (InterruptedException e) { e.printStackTrace(); + Thread.currentThread().interrupt(); } } timer = System.currentTimeMillis() + 1000; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index c04045ee9..77dddc0be 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -232,7 +232,6 @@ private void initStorageContexts(String[] storagePaths) throws StreamJobLogExcep LOG.info("Init the storage context: [" + StringUtils.join(storagePaths, ",") + "]"); for(String storagePath : storagePaths){ if (StringUtils.isNotBlank(storagePath)) { - // TODO the score of context this.storageContexts.add(new JobLogStorageContext(storagePath, 1.0)); } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index 0f2641185..989f424f3 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -144,10 +144,8 @@ public void close() { this.isShutdown.set(true); this.shutdownLock.lock(); try{ - if (activeThread.get() > 0) { - if (!this.canShutdown.await(5, TimeUnit.SECONDS)) { - LOG.warn("Shutdown the bucket: [{}] directly because the timeout of waiting", bucketName); - } + if (activeThread.get() > 0 && !this.canShutdown.await(5, TimeUnit.SECONDS)) { + LOG.warn("Shutdown the bucket: [{}] directly because the timeout of waiting", bucketName); } } catch (InterruptedException e) { // Ignore @@ -286,7 +284,7 @@ public long getBucketWriteTime() { }; } - private void notifyShutdown(){ + private synchronized void notifyShutdown(){ this.shutdownLock.lock(); try{ this.canShutdown.notifyAll(); diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java index c9bdaa857..afa210b04 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfExtRestfulApi.java @@ -19,7 +19,6 @@ import com.webank.wedatasphere.streamis.jobmanager.service.UserService; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.Message; -import org.apache.linkis.server.utils.ModuleUserUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index b3ce56371..c46c50161 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -206,10 +206,7 @@ public Message executeInspect(HttpServletRequest req, @RequestParam(value = "job Map inspectResultMap = new HashMap<>(); inspectResultMap.put("inspections", inspections); - inspectResult.forEach(inspect -> { - inspectResultMap.put(inspect.getInspectName(), inspect); - }); - + inspectResult.forEach(inspect -> inspectResultMap.put(inspect.getInspectName(), inspect)); result.data(String.valueOf(jobId), inspectResultMap); } return result; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java index 6a0660ffb..29433ad81 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java @@ -20,7 +20,6 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobService; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.Message; - import org.apache.linkis.server.utils.ModuleUserUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/UserServiceImpl.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/UserServiceImpl.java index 77b4fb618..022bc4ee5 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/UserServiceImpl.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/UserServiceImpl.java @@ -41,7 +41,7 @@ public List workspaceUserQuery(HttpServletRequest req,String workspaceId HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); headers.add("Cookie", req.getHeader("Cookie")); - HttpEntity httpEntity = new HttpEntity(headers); + HttpEntity httpEntity = new HttpEntity<>(headers); ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, httpEntity, Map.class); Map data = (Map) response.getBody().get("data"); List> workspaceUsers = data.get("workspaceUsers"); diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/rpc/StreamisJobManagerReceiver.scala b/streamis-jobmanager/streamis-jobmanager-server/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/rpc/StreamisJobManagerReceiver.scala index 89718ba85..01b6de8ef 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/rpc/StreamisJobManagerReceiver.scala +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/rpc/StreamisJobManagerReceiver.scala @@ -38,7 +38,7 @@ class StreamisJobManagerReceiver(jobService: StreamJobService) extends Receiver override def receive(message: Any, sender: Sender): Unit = { - + //nothing } override def receiveAndReply(message: Any, sender: Sender): Any = receiveAndReply(message, Duration(timeout, "seconds"), sender) diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java index ac452ca84..b41ca8c35 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java @@ -67,7 +67,7 @@ public Message upload(HttpServletRequest req, @RequestParam(name = "comment", required = false) String comment, @RequestParam(name = "updateWhenExists", required = false) boolean updateWhenExists, @RequestParam(name = "file") List files) throws UnsupportedEncodingException, FileException { - String username = ModuleUserUtils.getOperationUser(req, "upload project files");; + String username = ModuleUserUtils.getOperationUser(req, "upload project files"); if (StringUtils.isBlank(version)) { return Message.error("version is null"); } @@ -134,7 +134,6 @@ public Message versionList( HttpServletRequest req, @RequestParam(value = "fileN @RequestParam(value = "projectName",required = false) String projectName, @RequestParam(value = "pageNow",defaultValue = "1") Integer pageNow, @RequestParam(value = "pageSize",defaultValue = "20") Integer pageSize) { - String username = ModuleUserUtils.getOperationUser(req, "list project files version");; if (StringUtils.isBlank(projectName)) { return Message.error("projectName is null"); } diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/DateUtils.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/DateUtils.java index 355d0a5da..8cd97d9b7 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/DateUtils.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/DateUtils.java @@ -18,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -30,6 +31,7 @@ public class DateUtils { private static final String FORMAT_HH_MM_SS_S = "yyyy-MM-dd HH:mm:ss.S"; private static final String FORMAT_HH_MM = "yyyy-MM-dd HH:mm"; + private DateUtils(){} /** * contain T,Z format date time convert @@ -38,10 +40,9 @@ public class DateUtils { * @return * @throws Exception */ - public static String dateTimeTZConvert(String dateTime) throws Exception { + public static String dateTimeTZConvert(String dateTime) throws ParseException { Date date = new SimpleDateFormat(FORMAT_HH_T_Z).parse(dateTime); - String time = new SimpleDateFormat(FORMAT_HH_MM_SS).format(date); - return time; + return new SimpleDateFormat(FORMAT_HH_MM_SS).format(date); } /** diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/constant/ProjectUserPrivilegeEnum.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/constant/ProjectUserPrivilegeEnum.java index ce371cf47..2387dfe3c 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/constant/ProjectUserPrivilegeEnum.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/constant/ProjectUserPrivilegeEnum.java @@ -19,15 +19,9 @@ public int getRank() { return rank; } - public void setRank(int rank) { - this.rank = rank; - } public String getName() { return name; } - public void setName(String name) { - this.name = name; - } } diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/exception/ValidExceptionHander.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/exception/ValidExceptionHander.java index a7ef12d18..d8397976d 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/exception/ValidExceptionHander.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/exception/ValidExceptionHander.java @@ -25,7 +25,7 @@ public Message handle(MethodArgumentNotValidException e){ LOG.error("Request parameter validation exception", e); BindingResult bindingResult = e.getBindingResult(); StringBuilder stringBuilder = new StringBuilder(); - bindingResult.getFieldErrors().forEach((item) -> stringBuilder.append(item.getDefaultMessage()).append(";")); + bindingResult.getFieldErrors().forEach(item -> stringBuilder.append(item.getDefaultMessage()).append(";")); return Message.error("failed to validate request parameter, detail:"+stringBuilder.toString()); } diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectPrivilegeRestfulApi.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectPrivilegeRestfulApi.java index 75dc27933..a8e4ef3b6 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectPrivilegeRestfulApi.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectPrivilegeRestfulApi.java @@ -7,7 +7,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.math3.util.Pair; import org.apache.linkis.server.Message; -import org.apache.linkis.server.security.SecurityFilter; import org.apache.linkis.server.utils.ModuleUserUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestfulApi.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestfulApi.java index 609864857..90e390514 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestfulApi.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestfulApi.java @@ -11,7 +11,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.math3.util.Pair; import org.apache.linkis.server.Message; -import org.apache.linkis.server.security.SecurityFilter; import org.apache.linkis.server.utils.ModuleUserUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectPrivilegeUtils.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectPrivilegeUtils.java index 0fbfc5c79..2df77837d 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectPrivilegeUtils.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectPrivilegeUtils.java @@ -7,6 +7,11 @@ import java.util.List; public class StreamisProjectPrivilegeUtils { + + + private StreamisProjectPrivilegeUtils(){} + + public static List createStreamisProjectPrivilege(Long projectId, List users, int privilege){ List retList = new ArrayList<>(); if(CollectionUtils.isEmpty(users)){ diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectRestfulUtils.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectRestfulUtils.java index 050deb6f8..e24c8875e 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectRestfulUtils.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/utils/StreamisProjectRestfulUtils.java @@ -18,7 +18,7 @@ public static Message dealOk(String msg){ return Message.ok(msg); } - + private StreamisProjectRestfulUtils () {} @SafeVarargs public static Message dealOk(String msg, Pair... data){ diff --git a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/StreamisProjectServerReceiver.scala b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/StreamisProjectServerReceiver.scala index b74d9631b..5a2969c59 100644 --- a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/StreamisProjectServerReceiver.scala +++ b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/StreamisProjectServerReceiver.scala @@ -20,38 +20,13 @@ class StreamisProjectServerReceiver(projectService:StreamisProjectService) exten override def receive(message: Any, sender: Sender): Unit = { - + //nothing } override def receiveAndReply(message: Any, sender: Sender): Any = receiveAndReply(message, Duration.create(300, "seconds"), sender) override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = null -// message match { -// case createStreamProjectRequest: CreateStreamProjectRequest => -// Utils.tryCatch{ -// val streamisProject = projectService.createProject(createStreamProjectRequest) -// CreateStreamProjectResponse(0, streamisProject.getName, streamisProject.getId, "") -// }{ -// t => logger.error("failed to create project in streamis", t) -// CreateStreamProjectResponse(-1, createStreamProjectRequest.projectName, -1, t.getCause.getMessage) -// } -// case updateStreamProjectRequest: UpdateStreamProjectRequest => Utils.tryCatch{ -// projectService.updateProject(updateStreamProjectRequest) -// UpdateStreamProjectResponse(0, updateStreamProjectRequest.streamisProjectId, "") -// }{ -// t => logger.error(s"failed to update project ${updateStreamProjectRequest.projectName} in streamis",t) -// UpdateStreamProjectResponse(-1, updateStreamProjectRequest.streamisProjectId, t.getCause.getMessage) -// } -// -// case deleteStreamProjectRequest: DeleteStreamProjectRequest => Utils.tryCatch{ -// projectService.deleteProject(deleteStreamProjectRequest) -// DeleteStreamProjectResponse(0, deleteStreamProjectRequest.projectName, "") -// }{ -// t => logger.error(s"failed to update project ${deleteStreamProjectRequest.projectName} in streamis",t) -// DeleteStreamProjectResponse(-1, deleteStreamProjectRequest.projectName, t.getCause.getMessage) -// } -// case _ => -// } + From 329c6e4349191b8cfaa55049da8e68c87afb0be4 Mon Sep 17 00:00:00 2001 From: alexkun Date: Mon, 29 May 2023 13:32:00 +0800 Subject: [PATCH 171/219] 1. refactor flink manager ec params in LinkisFlinkManagerClient 2. fix some bugs --- .../linkis/conf/JobLauncherConfiguration.scala | 4 ++++ .../linkis/job/client/AbstractJobClient.scala | 17 ++++++++++++++++- .../job/client/LinkisFlinkManagerClient.scala | 11 +++-------- .../client/LinkisFlinkManagerJobClient.scala | 5 ++--- .../manager/SimpleFlinkJobLaunchManager.scala | 8 +++++++- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala index a5375cf49..76faf98e3 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala @@ -65,4 +65,8 @@ object JobLauncherConfiguration { val FLINK_MANAGER_MODE_KEY: CommonVars[String] = CommonVars("linkis.flink.manager.mode.key", "linkis.flink.manager.mode.on") val ENABLE_FLINK_MANAGER_EC_REFRESH_TASK: CommonVars[Boolean] = CommonVars("wds.streamis.flink.manager.ec.refresh.enable", true) + + val FLINKK_MANAGER_EXIT_TIME: CommonVars[Long] = CommonVars("wds.streamis.flink.manager.ec.expire.time.mills", 3600 * 1000) + + val LINKIS_EC_EXPIRE_TIME_KEY: CommonVars[String] = CommonVars("linkis.ec.expire.key", "wds.linkis.engineconn.max.free.time") } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/AbstractJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/AbstractJobClient.scala index c6981423a..219c49f7f 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/AbstractJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/AbstractJobClient.scala @@ -23,8 +23,9 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.exception.Fli import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.jobInfo.EngineConnJobInfo import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.operator.FlinkTriggerSavepointOperator import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.FlinkSavepoint +import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.computation.client.once.simple.SimpleOnceJob +import org.apache.linkis.computation.client.once.simple.{SimpleOnceJob, SubmittableSimpleOnceJob} import org.apache.linkis.computation.client.once.OnceJob /** @@ -49,8 +50,22 @@ abstract class AbstractJobClient(onceJob: OnceJob, jobInfo: JobInfo, stateManage override def getJobInfo(refresh: Boolean): JobInfo = { onceJob match { case simpleOnceJob: SimpleOnceJob => + Utils.tryCatch { jobInfo.setStatus(if (refresh && null != onceJob.getNodeInfo) onceJob.getNodeInfo .getOrDefault("nodeStatus", simpleOnceJob.getStatus).asInstanceOf[String] else simpleOnceJob.getStatus) + } { + case e: Exception => + val ec: ServiceInstance = simpleOnceJob.getEcServiceInstance + if (null != ec) { + if (e.getMessage.contains(s"Instance does not exist ServiceInstance(linkis-cg-engineconn, ${ec.getInstance}")) { + logger.warn(s"EC instance : ${ec.toString()} not exist, will set status to Failed.") + jobInfo.setStatus("Failed") + } + } else { + logger.error(s"EC instance of job : ${jobInfo.getId} is null, no need to get job status.") + throw e + } + } } jobInfo } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala index d0d52fe15..76285e088 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala @@ -62,7 +62,7 @@ class LinkisFlinkManagerClient extends FlinkManagerClient with Logging { initLabels.put(codeTypeLabel.getLabelKey, codeTypeLabel.getStringValue) initLabels.put(userCreatorLabel.getLabelKey, userCreatorLabel.getStringValue) initLabels.put(managerLabel.getLabelKey, managerLabel.getStringValue) - initLabels.put(SimpleOnceJobBuilder.ONCE_ENGINE_CONN_MODE_LABEL_KEY, SimpleOnceJobBuilder.ONCE_ENGINE_CONN_MODE_LABEL_VALUE) +// initLabels.put(SimpleOnceJobBuilder.ONCE_ENGINE_CONN_MODE_LABEL_KEY, SimpleOnceJobBuilder.ONCE_ENGINE_CONN_MODE_LABEL_VALUE) if (StringUtils.isNotBlank(JobLauncherConfiguration.FLINK_MANAGER_EC_TENANT.getValue)) { initLabels.put(LabelKeyConstant.TENANT_KEY, JobLauncherConfiguration.FLINK_MANAGER_EC_TENANT.getValue) } @@ -72,6 +72,7 @@ class LinkisFlinkManagerClient extends FlinkManagerClient with Logging { initProperties.put("flink.app.name", "FlinkManagerEC") initProperties.put(JobLauncherConfiguration.FLINK_MANAGER_MODE_KEY.getValue, true.toString) initProperties.put(AMConstant.EC_SYNC_START_KEY, true.toString) + initProperties.put(JobLauncherConfiguration.LINKIS_EC_EXPIRE_TIME_KEY.getValue, JobLauncherConfiguration.FLINKK_MANAGER_EXIT_TIME.getHotValue().toString) var askEngineConnAction = AskEngineConnAction .newBuilder() @@ -176,13 +177,7 @@ class LinkisFlinkManagerClient extends FlinkManagerClient with Logging { override def getFlinkManagerEngineConnInstance(): ServiceInstance = { if (null == ecInstance) { - LinkisFlinkManagerClient.ASK_EC_LOCK.synchronized { - if (null == ecInstance) { - val rs = getOrCreateLinkisManagerECAndECM() - ecInstance = rs._1 - ecmInstance = rs._2 - } - } + refreshManagerEC() } ecInstance } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala index 9cfc69c57..cb85e5ea8 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala @@ -56,11 +56,10 @@ class LinkisFlinkManagerJobClient(onceJob: OnceJob, jobInfo: JobInfo, stateManag override def getJobInfo(refresh: Boolean): JobInfo = { onceJob match { case simpleOnceJob: SimpleOnceJob => - if (JobConf.isCompleted(JobConf.linkisStatusToStreamisStatus(simpleOnceJob.getStatus))) { + if (JobConf.isCompleted(JobConf.linkisStatusToStreamisStatus(jobInfo.getStatus))) { jobInfo.setStatus(simpleOnceJob.getStatus) logger.info(s"Job : ${simpleOnceJob.getId} is completed, no need to get status from linkis.") - } - if (refresh && isDetachJob(jobInfo)) { + } else if (refresh && isDetachJob(jobInfo)) { jobInfo match { case engineConnJobInfo: EngineConnJobInfo => jobInfo.setStatus(getJobStatusWithRetry(engineConnJobInfo.getApplicationId)) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala index 68aa6aab1..7b47bc704 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala @@ -94,7 +94,13 @@ class SimpleFlinkJobLaunchManager extends FlinkJobLaunchManager { jobInfo } - override protected def createJobInfo(jobInfo: String): LinkisJobInfo = DWSHttpClient.jacksonJson.readValue(jobInfo, classOf[EngineConnJobInfo]) + override protected def createJobInfo(jobInfo: String): LinkisJobInfo = { + if (StringUtils.isNotBlank(jobInfo)) { + DWSHttpClient.jacksonJson.readValue(jobInfo, classOf[EngineConnJobInfo]) + } else { + null + } + } protected def fetchApplicationInfo(onceJob: OnceJob, jobInfo: EngineConnJobInfo): Unit = { val isDetach = JobClientType.DETACH.toString.equalsIgnoreCase(jobInfo.getClientType) From 298560fbff8a878a3abdbe5ff16d51dccbe248c8 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Mon, 29 May 2023 15:57:59 +0800 Subject: [PATCH 172/219] delete check on-off --- .../wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java index be860b051..f77a92a00 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/utils/HttpClientUtil.java @@ -43,7 +43,6 @@ public class HttpClientUtil { public static final CommonVars LINKIS_GATEWAY_URL = CommonVars.apply("wds.dss.gateway.url", ""); - public static final CommonVars LINKIS_HTTP_IS_REQUEST = CommonVars.apply("wds.linkis.http.is.request", ""); static { Registry socketFactoryRegistry = RegistryBuilder.create() @@ -146,7 +145,6 @@ public static boolean checkSystemAdmin(String token){ String linkisUrl= LINKIS_GATEWAY_URL.getValue()+ "/api/rest_j/v1/jobhistory/governanceStationAdmin"; Gson gson = BDPJettyServerHelper.gson(); boolean flag =false; - if (LINKIS_HTTP_IS_REQUEST.getValue().equals("true")) { try { String responseStr = get(null, linkisUrl, null, token).getResponseStr(); LinkisResponseData linkisResponseData = gson.fromJson(responseStr, LinkisResponse.class).getData(); @@ -154,7 +152,6 @@ public static boolean checkSystemAdmin(String token){ } catch (IOException e) { logger.error("Failed to request the Linkis system administrator interface"); } - } return flag; } } From 7b9da1ee4217f5092eb009bf3ee33970bca708d8 Mon Sep 17 00:00:00 2001 From: Liveipool <123653079@qq.com> Date: Wed, 31 May 2023 14:36:24 +0800 Subject: [PATCH 173/219] fix: bug --- web/src/apps/streamis/i18n/zh.json | 2 +- web/src/apps/streamis/module/jobList/index.vue | 12 ++++++++++-- web/vue.config.js | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/web/src/apps/streamis/i18n/zh.json b/web/src/apps/streamis/i18n/zh.json index 7af908c93..8c3eab690 100644 --- a/web/src/apps/streamis/i18n/zh.json +++ b/web/src/apps/streamis/i18n/zh.json @@ -48,7 +48,7 @@ "snapshotRestart": "快照重启", "editTags": "修改标签", "editTagsPlaceHolder": "请输入新标签", - "editTagsHint": "输入新标签,现有标签会被覆盖", + "editTagsHint": "输入新标签，现有标签会被覆盖", "startBtn": "启动", "configBtn": "配置", "saveBtn": "保存", diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index d19b3a55c..7cf4ebed5 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -304,7 +304,8 @@ + :title="$t('message.streamis.formItems.editTags')" + @on-visible-change="changeVisible">
@@ -667,9 +668,17 @@ export default { this.getJobList() }) }, + changeVisible(value) { + console.log('value: ', value); + if (!value) this.$refs['formValidate'].resetFields() + }, // 批量修改标签 async confirmEditTags() { this.editTagsLoading = true + if (this.formValidate.newTag.includes(' ')) { + this.editTagsLoading = false; + return this.$Message.error({ content: '标签不能有空格，仅支持大小写字母、数字、下划线、小数点、逗号' }); + } this.$refs['formValidate'].validate(async (valid) => { console.log('valid: ', valid); if (valid) { @@ -687,7 +696,6 @@ export default { this.$refs['formValidate'].resetFields(); this.showEditTags = false; this.editTagsLoading = false; - this.pageData.current = 1 this.getJobList() } catch { this.editTagsLoading = false; diff --git a/web/vue.config.js b/web/vue.config.js index b77872252..5bd80a7dc 100644 --- a/web/vue.config.js +++ b/web/vue.config.js @@ -124,8 +124,8 @@ module.exports = { port: 8080, proxy: { '/api': { - // target: 'http://sit.dss.bdp.weoa.com', - target: 'http://10.107.97.166:9188', + target: 'http://sit.dss.bdp.weoa.com', + // target: 'http://10.107.97.166:9188', // target: 'http://172.24.2.230:9400', changeOrigin: true, pathRewrite: { From 57860da98e7427c2dbfbebb03aff74870b545f18 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Wed, 31 May 2023 16:30:13 +0800 Subject: [PATCH 174/219] fix delete version file privilege --- .../streamis/projectmanager/dao/ProjectManagerMapper.java | 2 ++ .../projectmanager/dao/impl/ProjectManagerMapper.xml | 6 ++++++ .../restful/api/ProjectManagerRestfulApi.java | 5 +++-- .../projectmanager/service/ProjectManagerService.java | 2 ++ .../service/impl/ProjectManagerServiceImpl.java | 5 +++++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/ProjectManagerMapper.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/ProjectManagerMapper.java index 8eefbe92d..edec2de4e 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/ProjectManagerMapper.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/ProjectManagerMapper.java @@ -50,4 +50,6 @@ public interface ProjectManagerMapper { List selectProjectNamesByIds(List ids); String getProjectNameById(Long id); + + String getProjectNameByFileId(Long id); } diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/impl/ProjectManagerMapper.xml b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/impl/ProjectManagerMapper.xml index 6e017fdf2..6ca9ea274 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/impl/ProjectManagerMapper.xml +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/dao/impl/ProjectManagerMapper.xml @@ -108,4 +108,10 @@ SELECT j.project_name FROM linkis_stream_job_version_files jvf LEFT JOIN linkis_stream_job j ON jvf.job_id =j.id WHERE jvf.id = #{id} + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java index b41ca8c35..4d24ab614 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java @@ -173,8 +173,9 @@ public Message deleteVersion(HttpServletRequest req, @RequestParam(value = "ids" idList.add(Long.parseLong(s)); } } - List projectNames = projectManagerService.getProjectNames(idList); - if (!projectPrivilegeService.hasEditPrivilege(req,projectNames)) { + String projectName = projectManagerService.getProjectNameByFileId(Long.valueOf(ids)); +// List projectNames = projectManagerService.getProjectNames(idList); + if (!projectPrivilegeService.hasEditPrivilege(req,projectName)) { return Message.error("the current user has no operation permission"); } diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/ProjectManagerService.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/ProjectManagerService.java index 9ff146678..6df103c49 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/ProjectManagerService.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/ProjectManagerService.java @@ -48,4 +48,6 @@ public interface ProjectManagerService { List getProjectNames(List ids); String getProjectNameById(Long id); + + String getProjectNameByFileId(Long id); } diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java index 114370837..ab54c36b8 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java @@ -141,4 +141,9 @@ public List getProjectNames(List ids) { public String getProjectNameById(Long id) { return projectManagerMapper.getProjectNameById(id); } + + @Override + public String getProjectNameByFileId(Long id) { + return projectManagerMapper.getProjectNameByFileId(id); + } } From 7f5c75634ec26194d280a5d268e32aa7fa29b72f Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Wed, 31 May 2023 16:33:02 +0800 Subject: [PATCH 175/219] Add a snapshot tip failure --- .../jobmanager/restful/api/JobRestfulApi.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 734921b4d..68102e382 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -278,10 +278,16 @@ public Message executeInspect(HttpServletRequest req, @RequestParam(value = "job return Message.error("Fail to inspect job " + jobId + " of the execution(任务执行前检查失败), message: " + e.getMessage()); } - Map inspectResultMap = new HashMap<>(); - inspectResultMap.put("inspections", inspections); + HashMap inspectResultMap = new HashMap<>(); inspectResult.forEach(inspect -> inspectResultMap.put(inspect.getInspectName(), inspect)); - result.data(String.valueOf(jobId), inspectResultMap); + if (!inspectResultMap.containsKey("snapshot")){ + inspections.add("snapshot"); + JobSnapshotInspectVo jobSnapshotInspectVo =new JobSnapshotInspectVo(); + jobSnapshotInspectVo.setPath("任务无法快照"); + inspectResultMap.put("snapshot",jobSnapshotInspectVo); + } + inspectResultMap.put("inspections", inspections); + result.setData(inspectResultMap); } return result; } From 2cee43e486fd27a51899392ee5961934300c3c30 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Wed, 31 May 2023 18:14:28 +0800 Subject: [PATCH 176/219] case-sensitive --- .../streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index de971870b..c5ced7eb9 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -128,7 +128,7 @@ AND aa.`create_by` = #{createBy} - AND aa.`label` like concat('%', #{label}, '%') + AND binary aa.`label` like concat('%', #{label}, '%') AND EXISTS (SELECT 1 FROM linkis_stream_project_privilege lspp WHERE lspp.project_id =aa.project_id From ee96712ad32809af8eecd63102c86bfedfe62e34 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Wed, 31 May 2023 18:18:10 +0800 Subject: [PATCH 177/219] fix md5 --- .../impl/ProjectManagerServiceImpl.java | 2 +- .../projectmanager/utils/MD5Utils.java | 76 ++++++++++--------- 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java index ab54c36b8..95b917d97 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java @@ -51,7 +51,7 @@ public class ProjectManagerServiceImpl implements ProjectManagerService, Streami @Override @Transactional(rollbackFor = Exception.class) public void upload(String username, String fileName, String version, String projectName, String filePath,String comment) throws JsonProcessingException { - String fileMd5 = MD5Utils.getMD5(new File(filePath)); + String fileMd5 = MD5Utils.getMD5(filePath); Map result = bmlService.upload(username, filePath); ProjectFiles projectFiles = new ProjectFiles(); projectFiles.setFileName(fileName); diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/MD5Utils.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/MD5Utils.java index 86346630f..1b0522239 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/MD5Utils.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/utils/MD5Utils.java @@ -3,48 +3,56 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; +import java.io.*; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; public class MD5Utils { - /** - * 获取一个文件的md5值(可处理大文件) - * @return md5 value - */ - public static String getMD5(File file) { - FileInputStream fileInputStream = null; + public static String getMD5(String filePath) { + byte[] key = getBytes(filePath); + String cacheKey; try { - MessageDigest MD5 = MessageDigest.getInstance("MD5"); - fileInputStream = new FileInputStream(file); - byte[] buffer = new byte[8192]; - int length; - while ((length = fileInputStream.read(buffer)) != -1) { - MD5.update(buffer, 0, length); - } - return new String(Hex.encodeHex(MD5.digest())); - } catch (Exception e) { - e.printStackTrace(); - return null; - } finally { - try { - if (fileInputStream != null){ - fileInputStream.close(); - } - } catch (IOException e) { - e.printStackTrace(); + final MessageDigest mDigest = MessageDigest.getInstance("MD5"); + mDigest.update(key); + cacheKey = bytesToHexString(mDigest.digest()); + } catch (NoSuchAlgorithmException e) { + cacheKey = String.valueOf(Arrays.hashCode(key)); + } + return cacheKey; + } + + private static String bytesToHexString(byte[] bytes) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < bytes.length; i++) { + String hex = Integer.toHexString(0xFF & bytes[i]); + if (hex.length() == 1) { + sb.append('0'); } + sb.append(hex); } + return sb.toString(); } - /** - * 求一个字符串的md5值 - * @param target 字符串 - * @return md5 value - */ - public static String MD5(String target) { - return DigestUtils.md5Hex(target); + private static byte[] getBytes(String filePath) { + byte[] buffer = null; + try { + File file = new File(filePath); + FileInputStream fis = new FileInputStream(file); + ByteArrayOutputStream bos = new ByteArrayOutputStream(1000); + byte[] b = new byte[1000]; + int n; + while ((n = fis.read(b)) != -1) { + bos.write(b, 0, n); + } + fis.close(); + bos.close(); + buffer = bos.toByteArray(); + } catch (IOException e) { + e.printStackTrace(); + } + return buffer; } + } From 9ea8207860d29e5376811633b8e5abf8112c42fd Mon Sep 17 00:00:00 2001 From: alexkun Date: Wed, 31 May 2023 23:06:39 +0800 Subject: [PATCH 178/219] 1. fix getJobInfo bug 2. refactor manageMode in JobDetailsVo --- .../job/client/LinkisFlinkManagerJobClient.scala | 2 +- .../jobmanager/manager/dao/impl/StreamJobMapper.xml | 4 ++-- .../streamis/jobmanager/manager/entity/StreamJob.java | 11 +++++++++++ .../jobmanager/manager/entity/vo/JobDetailsVo.java | 10 +++------- .../manager/service/DefaultStreamTaskService.scala | 2 +- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala index cb85e5ea8..27b96864b 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala @@ -56,7 +56,7 @@ class LinkisFlinkManagerJobClient(onceJob: OnceJob, jobInfo: JobInfo, stateManag override def getJobInfo(refresh: Boolean): JobInfo = { onceJob match { case simpleOnceJob: SimpleOnceJob => - if (JobConf.isCompleted(JobConf.linkisStatusToStreamisStatus(jobInfo.getStatus))) { + if (StringUtils.isNotBlank(jobInfo.getStatus) && JobConf.isCompleted(JobConf.linkisStatusToStreamisStatus(jobInfo.getStatus))) { jobInfo.setStatus(simpleOnceJob.getStatus) logger.info(s"Job : ${simpleOnceJob.getId} is completed, no need to get status from linkis.") } else if (refresh && isDetachJob(jobInfo)) { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index de971870b..6a40588a1 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -76,7 +76,7 @@ - `id`,`project_name`, `workspace_name`,`name`,`create_by`, `create_time`,`label`,`description`,`submit_user`,`job_type`, `current_version` + `id`,`project_name`, `workspace_name`,`name`,`create_by`, `create_time`,`label`,`description`,`submit_user`,`job_type`, `current_version`, `status` @@ -101,7 +101,7 @@