Skip to content

W
wj-datacenter-platform
Commit e3274d28 authored by hanbing's avatar hanbing
Browse files
Options

数据中心在线取样本数据采集

parent 389fdd85
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 704 additions and 12 deletions
wj-datacenter-service/src/main/java/net/wanji/datacenter/config/EventSaveConfiguration.java 0 → 100644
View file @ e3274d28
package net.wanji.datacenter.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* # 事件数据集文件保存配置
*/
@ConfigurationProperties(prefix = "eventsave")
@Component
@Data
public class EventSaveConfiguration {
String rootFolder;
String city;
String subType;
String nvrurl;
String positiveSample;
String negativeSample;
String trackJson;
String lightJson;
String evidence;
String image;
String video;
String vision;
String labelJson;
}
wj-datacenter-service/src/main/java/net/wanji/datacenter/constant/EventAbnormalEnum.java
View file @ e3274d28
......@@ -191,6 +191,24 @@ public enum EventAbnormalEnum {
}
);
//路口事件类型
public static final List<EventAbnormalEnum> NotB5Events = Arrays.asList(
new EventAbnormalEnum[] {
EventAbnormalEnum.ACCIDENT_YS,
EventAbnormalEnum.ACCIDENT_ONE_YS,
EventAbnormalEnum.ACCIDENT_TWO_YS,
EventAbnormalEnum.ACCIDENT_MORE_YS,
EventAbnormalEnum.NO_CONGEST,
EventAbnormalEnum.LIGHT_CONGEST,
EventAbnormalEnum.MODERATE_CONGEST,
EventAbnormalEnum.HEAVY_CONGEST,
EventAbnormalEnum.PHASE_EMPTY,
EventAbnormalEnum.CROSS_UNBALANCE,
EventAbnormalEnum.CROSS_OVERFLOW,
EventAbnormalEnum.CROSS_DEADLOCK
}
);
// 机动车事件
public static final List<EventAbnormalEnum> VehicleEvents = Arrays.asList(new EventAbnormalEnum[]{
EventAbnormalEnum.ILLEGALSTOP,
......@@ -416,4 +434,13 @@ public enum EventAbnormalEnum {
}
return list;
}
// 非 B5 事件
public static List<String> getNotB5List() {
List<String> list = new ArrayList<>();
for (EventAbnormalEnum eventEnum : EventAbnormalEnum.NotB5Events) {
list.add(eventEnum.getType());
}
return list;
}
}
wj-datacenter-service/src/main/java/net/wanji/datacenter/mapper/EventInfoMapper.java 0 → 100644
View file @ e3274d28
package net.wanji.datacenter.mapper;
import com.baomidou.dynamic.datasource.annotation.DS;
import net.wanji.datacenter.pojo.po.EventInfoPO;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* @author Kent HAN
* @date 2024/3/4 10:04
*/
@Repository
@DS("holo")
public interface EventInfoMapper {
List<EventInfoPO> selectLastTenMinutes();
}
wj-datacenter-service/src/main/java/net/wanji/datacenter/mapper/HoloMapper.java 0 → 100644
View file @ e3274d28
package net.wanji.datacenter.mapper;
import com.baomidou.dynamic.datasource.annotation.DS;
import net.wanji.datacenter.pojo.po.DeviceCameraPO;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
@DS("holo")
public interface HoloMapper {
List<DeviceCameraPO> selectVideoUrlsByCrossId(String crossId);
}
\ No newline at end of file
wj-datacenter-service/src/main/java/net/wanji/datacenter/pojo/po/DeviceCameraPO.java 0 → 100644
View file @ e3274d28
package net.wanji.datacenter.pojo.po;
import lombok.Data;
/**
* holo_roadnet 数据库 t_device_camera 实体类
*/
@Data
public class DeviceCameraPO {
String video_url;
String monitor_direct;
String monitor_channel;
}
wj-datacenter-service/src/main/java/net/wanji/datacenter/pojo/po/EventInfoPO.java 0 → 100644
View file @ e3274d28
package net.wanji.datacenter.pojo.po;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.Date;
/**
* @author Kent HAN
* @date 2024/3/4 10:18
*/
@Data
public class EventInfoPO {
/** 主键ID */
@ApiModelProperty(name = "主键ID",notes = "")
private Long oid ;
/** 车牌号 */
@ApiModelProperty(name = "车牌号",notes = "")
private String plateNo ;
/** 参与者类别 1：机动车 2：非机动车 3：行人 */
@ApiModelProperty(name = "参与者类别 1：机动车 2：非机动车 3：行人",notes = "")
private String objectType ;
/** 可信度 */
@ApiModelProperty(name = "可信度",notes = "")
private Integer confidence ;
/** 检测时间 */
@ApiModelProperty(name = "检测时间",notes = "")
private Date detectTime ;
/** 事件等级：1扣分 2 罚款 3警告 0 未知 */
@ApiModelProperty(name = "事件等级：1扣分 2 罚款 3警告 0 未知",notes = "")
private Integer grade ;
/** 事件地点描述 */
@ApiModelProperty(name = "事件地点描述",notes = "")
private String placeDesc ;
/** 发生地点经度 */
@ApiModelProperty(name = "发生地点经度",notes = "")
private Double lng ;
/** 发生地点纬度 */
@ApiModelProperty(name = "发生地点纬度",notes = "")
private Double lat ;
/** 事件一级类别 事件一级类别 1：机动车事件 2：非机动车事件 3：行人事件 */
@ApiModelProperty(name = "事件一级类别 事件一级类别 1：机动车事件 2：非机动车事件 3：行人事件",notes = "")
private String category ;
/** 二级类别 */
@ApiModelProperty(name = "二级类别",notes = "")
private String type ;
/** 事件发生时间 */
@ApiModelProperty(name = "事件发生时间",notes = "")
private Date startTime ;
/** 事件结束时间 */
@ApiModelProperty(name = "事件结束时间",notes = "")
private Date endTime ;
/** 事件持续时长 */
@ApiModelProperty(name = "事件持续时长",notes = "")
private Integer duration ;
/** 事件来源 */
@ApiModelProperty(name = "事件来源",notes = "")
private String source ;
/** 入库时间 */
@ApiModelProperty(name = "入库时间",notes = "")
private Date ruksj ;
/** 车道id */
@ApiModelProperty(name = "车道id",notes = "")
private String laneId ;
/** 路段id */
@ApiModelProperty(name = "路段id",notes = "")
private String rid ;
/** 渠化id */
@ApiModelProperty(name = "渠化id",notes = "")
private String segmentId ;
/** 路口id */
@ApiModelProperty(name = "路口id",notes = "")
private String crossId ;
/** 关联的事件摄像头信息ID */
@ApiModelProperty(name = "关联的事件摄像头信息ID",notes = "")
private Long cameraOid ;
/** 事件序列号作为主键 */
@ApiModelProperty(name = "事件序列号作为主键",notes = "")
private String eventSerialNumber ;
/** */
@ApiModelProperty(name = "",notes = "")
private Character dataStatus ;
/** */
@ApiModelProperty(name = "",notes = "")
private String globalId ;
/** */
@ApiModelProperty(name = "",notes = "")
private Integer stationId ;
/** */
@ApiModelProperty(name = "",notes = "")
private String eventId ;
}
wj-datacenter-service/src/main/java/net/wanji/datacenter/pojo/vo/B5LabelVO.java 0 → 100644
View file @ e3274d28
package net.wanji.datacenter.pojo.vo;
import lombok.Data;
import java.util.Date;
import java.util.List;
/**
* B5 数据集，标签实体类
*/
@Data
public class B5LabelVO {
Integer eventType;
List<Integer> targetIdList;
Date startTime;
Date endTime;
}
wj-datacenter-service/src/main/java/net/wanji/datacenter/util/TimeBasedKafkaConsumer.java 0 → 100644
View file @ e3274d28
package net.wanji.datacenter.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
import java.util.Properties;
@Component
public class TimeBasedKafkaConsumer {
private static String lightStatusTopic;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@Value("${kafka.topics.lightStatusTopic}")
public void setLightStatusTopic(String lightStatusTopic) {
TimeBasedKafkaConsumer.lightStatusTopic = lightStatusTopic;
}
public static void consumeMessagesAroundTimestamp(String bootstrapServers, String topic, long startTimeStamp,
long endTimeStamp, String kafkaSavepath, String fileName) {
Properties props = new Properties();
props.put("bootstrap.servers", bootstrapServers);
props.put("group.id", "time-based-consumer");
props.put("key.deserializer", StringDeserializer.class.getName());
props.put("value.deserializer", StringDeserializer.class.getName());
File file = new File(kafkaSavepath + File.separator + fileName);
try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
BufferedWriter writer = new BufferedWriter(new FileWriter(file, true))) {
TopicPartition partition = new TopicPartition(topic, 0);
consumer.assign(Collections.singletonList(partition));
HashMap<TopicPartition, Long> timestampToSearch = new HashMap<>();
timestampToSearch.put(partition, startTimeStamp);
long startOffset = consumer.offsetsForTimes(timestampToSearch).get(partition).offset();
// 开始消费
consumer.seek(partition, startOffset);
boolean keepConsuming = true;
int emptyPollCount = 0; // 记录空轮询次数的计数器
int maxEmptyPolls = 10; // 设置最大空轮询次数
while (keepConsuming) {
ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
if (records.isEmpty()) {
emptyPollCount++; // 如果没有记录，增加空轮询计数
if (emptyPollCount >= maxEmptyPolls) {
// 如果达到最大空轮询次数，退出循环
break;
}
} else {
emptyPollCount = 0; // 如果有记录，重置空轮询计数器
for (ConsumerRecord<String, String> record : records) {
saveRecordToDisk(record, writer);
if (record.timestamp() > endTimeStamp) {
keepConsuming = false;
break;
}
}
}
}
} catch (IOException e) {
System.err.println("Error writing to file: " + e.getMessage());
}
}
private static void saveRecordToDisk(ConsumerRecord<String, String> record, BufferedWriter writer) throws IOException {
String value = record.value();
writer.write(value);
writer.newLine();
writer.flush();
}
}
wj-datacenter-service/src/main/resources/application-local.yaml
View file @ e3274d28
......@@ -32,6 +32,12 @@ spring:
username: root
password: Wanji300552
driver-class-name: com.mysql.cj.jdbc.Driver
holo:
type: com.alibaba.druid.pool.DruidDataSource
url: jdbc:mysql://10.102.1.182:3306/holo_roadnet_jn?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'&useSSL=false&useCursorFetch=true
username: root
password: Wanji300552
driverClassName: com.mysql.cj.jdbc.Driver
kafka:
bootstrap-servers: 43.5.252.72:9092
producer:
......@@ -101,4 +107,34 @@ threadPoolConfig:
kafka-consumer:
lanePeriodicDataTopic: MZ050001LanePeriodicData
crossSnapshotDataTopic: MZ050001CrossSnapshotData
laneSnapshotDataTopic: MZ050001LaneSnapshotData
\ No newline at end of file
laneSnapshotDataTopic: MZ050001LaneSnapshotData
# 事件数据集文件保存配置
eventsave:
rootFolder: /data/Abnormal_event_dataset
city: jinan
subType: "0"
nvrurl: rtsp://admin:wanji123@37.12.182.33:554/cam/playback
positiveSample: 正样本
negativeSample: 负样本
trackJson: track.json
lightJson: light.json
evidence: 取证文件夹
image: 图像
video: 视频
vision: 可视化
labelJson: label.json
kafka:
producer:
bootstrap.servers: 37.12.182.31:9092 # kafka连接地址
acks: 0
retries: 0
batch.size: 102400
linger: 0
max.block.ms: 0
topics:
matchResultTopic: JNMatchResultData # 全域匹配结果-topic
topicPrefix: JN #主题前缀
matchResultMiniData: JNMatchResultMiniData # 全域匹配结果缩减版（展示端用）-topic
lightStatusTopic: cross_lights_status # 灯态数据
\ No newline at end of file
wj-datacenter-service/src/main/resources/mapper/EventInfoMapper.xml 0 → 100644
View file @ e3274d28
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="net.wanji.datacenter.mapper.EventInfoMapper">
<sql id="Base_Column_List">
oid,plate_no,object_type,confidence,detect_time,grade,place_desc,lng,lat,category,type,start_time,end_time,
duration,source,ruksj,lane_id,rid,segment_id,cross_id,camera_oid,event_serial_number,data_status,global_id,
station_id,event_id
</sql>
<select id="selectLastTenMinutes" resultType="net.wanji.datacenter.pojo.po.EventInfoPO">
SELECT <include refid="Base_Column_List"/>
FROM t_event_info
WHERE start_time >= NOW() - INTERVAL 10 MINUTE
</select>
</mapper>
wj-datacenter-service/src/main/resources/mapper/HoloMapper.xml 0 → 100644
View file @ e3274d28
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="net.wanji.datacenter.mapper.HoloMapper">
<select id="selectVideoUrlsByCrossId" resultType="net.wanji.datacenter.pojo.po.DeviceCameraPO">
select video_url, equipment_direct, monitor_channel
from t_device_camera
where cross_id = #{crossId}
</select>
</mapper>
\ No newline at end of file
wj-realtime-computing/src/main/resources/config.properties
View file @ e3274d28
config_name=config_mz.properties
\ No newline at end of file
config_name=config_jn.properties
\ No newline at end of file
wj-realtime-computing/src/main/resources/data/cross_exit_info.data
View file @ e3274d28
12F6L08J630:12F6L08J63012F6S08J7P00:112.96313284188527,28.18318575342956;112.96303313946666,28.183194356219847;112.96305539467383,28.183428386959946;112.96315227754664,28.18342099486973;112.96313284188527,28.18318575342956
12F6L08J630:12F6L08J63012F7O08J5U00:112.96354410000906,28.18283950134074;112.96352853848853,28.18271853213667;112.9633037242585,28.182742264061066;112.96331953166877,28.18286652039383;112.96354410000906,28.18283950134074
12F6L08J630:12F6L08J63012F6F08J4600:112.96292323382805,28.182416762776572;112.96282914613639,28.182429462626725;112.96285131325381,28.182654093569635;112.962945350963,28.182641866754853;112.96292323382805,28.182416762776572
12F6S08J7P0:12F6S08J7P012F6S08J8T00:112.96379954396568,28.188516678001562;112.96370564700435,28.188527356789084;112.96373122467124,28.188746864962972;112.96382613127733,28.18873559293259;112.96379954396568,28.188516678001562
12F6S08J7P0:12F6S08J7P012F8308J7E00:112.96418312352651,28.188152488433946;112.9641502272407,28.18806876123086;112.9639356562015,28.188136413689957;112.96396894420842,28.188225208365164;112.96418312352651,28.188152488433946
12F6S08J7P0:12F6S08J7P012F6L08J6300:112.96357638149044,28.18786559221101;112.9634794865813,28.187875948798126;112.96350842973068,28.18810139099276;112.96360563818898,28.18808988000444;112.96357638149044,28.18786559221101
12F6S08J8T0:12F6S08J8T012F7L08J8S00:112.96418244046701,28.19192859057195;112.96417684807375,28.191833814493464;112.96395118642712,28.191839035877035;112.96395631087961,28.191933726561583;112.96418244046701,28.19192859057195
12F6S08J8T0:12F6S08J8T012F6S08J7P00:112.96374376109364,28.19149557373418;112.96364952758402,28.19148400543331;112.9636140217438,28.191706917471826;112.96370808697927,28.191718930682264;112.96374376109364,28.19149557373418
12F6S08J8T0:12F6S08J8T012F6N08JA100:112.96379087784844,28.192091194225977;112.96369664433882,28.192079625925107;112.9636611384986,28.192302537963624;112.96375520373407,28.19231455117406;112.96379087784844,28.192091194225977
\ No newline at end of file
13NID0B5RM0:13NID0B5RM013NHC0B5R400:117.09604169736885,36.644790032250896;117.09608051785587,36.6448244460881;117.0962419846417,36.6446995386553;117.09620449671407,36.64466409552374;117.09604169736885,36.644790032250896
13NID0B5RM0:13NID0B5RM013NH20B5RH00:117.09624256906865,36.64489034135067;117.09624592651616,36.64496882168675;117.09647304659556,36.64496025015195;117.09646623060874,36.644884251898674;117.09624256906865,36.64489034135067
13NGH0B5RC0:13NGH0B5RC013NF80B5QN00:117.08953880238352,36.64388059783646;117.08950438164997,36.64395233609795;117.08971033045316,36.644030528158645;117.08974645844097,36.64395897597499;117.08953880238352,36.64388059783646
13NID0B5RM0:13NID0B5RM013NI00B5RM00:117.0955355076057,36.64498427650557;117.09553346280966,36.645071521137126;117.09576213835423,36.64507503430101;117.09576377639532,36.644988033569696;117.0955355076057,36.64498427650557
13NGH0B5RC0:13NGH0B5RC013NEP0B5QJ00:117.0898749695255,36.643852310026844;117.08999168758082,36.64389566386026;117.09006533736702,36.643726045334844;117.0899566223769,36.643682252619385;117.0898749695255,36.643852310026844
13NGH0B5RC0:13NGH0B5RC013NI00B5RM00:117.09019785998227,36.644004177469945;117.09016996276503,36.64408081744988;117.09038057675839,36.6441482957196;117.09040920390312,36.644076046259094;117.09019785998227,36.644004177469945
13NF80B5QN0:13NF80B5QN013NED0B5Q900:117.08566152826947,36.64176235952611;117.08560972676949,36.64184040257543;117.08578777591498,36.64195684491826;117.08583397273655,36.64187414171029;117.08566152826947,36.64176235952611
13NF80B5QN0:13NF80B5QN013NGH0B5RC00:117.08605943390197,36.641911479079525;117.08599681823961,36.64198347196667;117.08617136408513,36.642100432473406;117.0862344119634,36.64202920541093;117.08605943390197,36.641911479079525
13NF80B5QN0:13NF80B5QN013NDG0B5RI00:117.08589754298806,36.64204909701344;117.08583946469253,36.642018242918944;117.08574434250956,36.64219449309183;117.08579989280231,36.642216304249715;117.08589754298806,36.64204909701344
13NF80B5QN0:13NF80B5QN013NDT0B5Q900:117.08609025384176,36.64171158042417;117.0860452683286,36.64167647809194;117.08589676368292,36.64179832731728;117.08594619058222,36.64183558031174;117.08609025384176,36.64171158042417
13NGH0B5RC0:13NGH0B5RC013NEH0B5RJ00:117.09006942695913,36.64421338839392;117.08992142320416,36.64416398791302;117.089869036946,36.64434016574915;117.09000808307756,36.64438106167019;117.09006942695913,36.64421338839392
13NI00B5RM0:13NI00B5RM013NGH0B5RC00:117.09423195012252,36.64493383820295;117.09422377093831,36.64501085885425;117.09444590928958,36.6450299635677;117.09445770644335,36.64495343331372;117.09423195012252,36.64493383820295
13NI00B5RM0:13NI00B5RM013NG40B5SK00:117.0948077755471,36.64508885308461;117.09466741966627,36.64509218048696;117.09467498926716,36.64527531914366;117.09481130900397,36.645269866354184;117.0948077755471,36.64508885308461
13NI00B5RM0:13NI00B5RM013NID0B5RM00:117.09485437114998,36.64487766904978;117.094851951221,36.64496660143979;117.09507645089205,36.64497132613058;117.09508054048415,36.64488544469639;117.09485437114998,36.64487766904978
\ No newline at end of file
wj-realtime-computing/src/main/resources/spring-mybatis.xml
View file @ e3274d28
......@@ -10,7 +10,7 @@
<!-- 加载配置文件 -->
<!--<context:property-placeholder location="classpath:db.properties"/>-->
<context:property-placeholder location="classpath:db/mengzi/db.properties"/>
<context:property-placeholder location="classpath:db/jinan/db.properties"/>
<!-- 配置数据源 -->
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" init-method="init" destroy-method="close">
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment