feat: yolo数据标注

js/yolo 数据集工具/parse.js

@@ -0,0 +1,430 @@
+// 1. 读取指定目录文件, 获取文件内容.
+// 2. 解析内容, 获取图片的标注信息
+// 3. 统计标注信息
+
+const fs = require('fs');
+const path = require('path');
+const readline = require('readline');
+
+function readFile(filePath) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(filePath, 'utf-8', (err, data) => {
+            if (err) return reject(err);
+            resolve(data);
+        });
+    })
+}
+
+function readDir(filePath) {
+    return new Promise((resolve, reject) => {
+        fs.readdir(filePath, (err, files) => {
+            if (err) return reject(err);
+            resolve(files);
+        });
+    })
+}
+
+function parseYolo(fileData) {
+    // 解析文件内容, 获取图片的标注信息
+    // 类型 x坐标 y坐标 宽度 高度
+    // 1 0.391297 0.095892 0.280578 0.179688
+    let result = [];
+    let lines = fileData.split('\n');
+    lines.forEach(line => {
+        // 判断是否为空行
+        if (line.trim() === '') return;
+        let [type, x, y, width, height] = line.split(' ');
+        result.push({ type, x, y, width, height });
+    });
+    return result;
+}
+
+
+
+async function writeFile(filePath, fileData) {
+    return new Promise((resolve, reject) => {
+        // 判断父级目录是否存在
+        let dirPath = path.dirname(filePath);
+        if (!fs.existsSync(dirPath)) {
+            fs.mkdirSync(dirPath, {recursive: true });
+        }
+
+        fs.writeFile(filePath, fileData, (err, data) => {
+            if (err) return reject(err);
+            resolve(data);
+        });
+    })
+}
+
+async function mvFile(filePath, targetPath) {
+    return new Promise((resolve, reject) => {
+        // 判断父级目录是否存在
+        let dirPath = path.dirname(filePath);
+        if (!fs.existsSync(dirPath)) {
+            fs.mkdirSync(dirPath, {recursive: true });
+        }
+        fs.rename(filePath, targetPath, (err, data) => {
+            if (err) return reject(err);
+            resolve(data);
+        });
+    })
+}
+
+async function cpFile(filePath, targetPath) {
+    return new Promise((resolve, reject) => {
+        // 判断父级目录是否存在
+        let dirPath = path.dirname(targetPath);
+        if (!fs.existsSync(dirPath)) {
+            console.log(`路径${dirPath} 不存在`)
+            fs.mkdirSync(dirPath, {recursive: true });
+            console.log('路径不存在')
+        }
+        // console.log('路径不存在')
+
+        fs.copyFile(filePath, targetPath, (err, data) => {
+            if (err) return reject(err);
+            resolve(data);
+        });
+    })
+}
+
+
+// 删除指定目录
+function rmDir(path) {
+    return new Promise((resolve, reject) => {
+        fs.rmdir(path, { recursive: true }, (err, data) => {
+            if (err) return reject(err);
+            resolve(data);
+        });
+    })
+}
+
+
+
+
+
+
+
+
+
+
+
+
+const dimensionType = {
+    "-1": "无标注",
+    "0": "吊车",
+    "1": "塔吊",
+    "2": "烟火",
+    "3": "施工机械",
+    "4": "导线异物",
+    "5": "烟雾"
+}
+
+
+async function main(){
+
+    // let labelsPath = `E:\\图库\\ai\\epower_v2\\newLabels`
+    // let labelsPath = `E:\\图库\\ai\\epower_v2\\处理labels`
+    // let labelsPath = `E:\\图库\\ai\\epower_v2\\labels`
+    let labelsPath = `E:\\图库\\ai\\epower\\all_tmpLabels_2`
+
+    let imagesPath = `E:\\图库\\ai\\epower\\images`
+    let emptyPath = `E:\\图库\\ai\\epower\\empty`
+    let imageExts = ['jpg']
+    let logsPath = `E:\\图库\\ai\\epower\\logs`
+
+    // 数据转移目录
+    let transferPath = `E:\\图库\\ai\\epower\\parse`
+    // 清除数据转移目录
+    // 判断数据转移目录是否存在
+    if (fs.existsSync(transferPath)) {        // 删除数据转移目录
+        await rmDir(transferPath);
+    }
+
+    let countMap = {};
+
+    let logStrArr = [];
+    let resultStrArr = [];
+    // 获取文件列表
+    let labelFiles = await readDir(labelsPath);
+    let imagesFiles = await readDir(imagesPath);
+
+    // 获取当前时间
+    let now = new Date();
+    let nowStr = `${now.getFullYear()}-${now.getMonth()+1}-${now.getDate()} ${now.getHours()}点${now.getMinutes()}分${now.getSeconds()}秒`;
+    resultStrArr.push(`[I] [START] 开始统计分析数据标注信息 ${nowStr}`);
+    resultStrArr.push(`[I] 原标注信息路径: ${labelsPath}`);
+    resultStrArr.push(`[I] 原图片文件路径: ${labelsPath}`);
+    resultStrArr.push(`[I] 输出文件路径: ${labelsPath}`);
+    resultStrArr.push(`[I] 输出文件路径: ${logsPath}`);
+    resultStrArr.push(`[I] 标注文件数量: ${labelFiles.length}`);
+    resultStrArr.push(`[I] 图片文件数量: ${imagesFiles.length}`);
+    resultStrArr.push(`[I] 图片后缀: ${imageExts}`);
+
+    // 判断labels中是否包含 classes.txt. 如果有则 将文件 复制至 transferPath 目录
+    if (fs.existsSync(path.join(labelsPath, 'classes.txt'))) {
+        await cpFile(path.join(labelsPath, 'classes.txt'), path.join(transferPath, 'classes.txt'))
+    }
+
+    // 将 resultArr 的内容, 转移至 logStrArr
+    resultStrArr.forEach(item => {
+        logStrArr.push(item);
+    });
+
+    for (const [i, labelFile] of labelFiles.entries()) {
+
+        let filePath = path.join(labelsPath, labelFile);
+        // 获取对应的图片路径, 与 label 文件的 名称相同
+        let labelName = labelFile.replace('.txt', '');
+        // 读取文件内容
+        let fileData = await readFile(filePath);
+        // 解析文件内容
+        let result = parseYolo(fileData);
+
+        // 图片后缀可能为 jpg, png 等
+        let imagePath = '';
+        let imageName = labelName;
+        for (let j = 0; j < imageExts.length; j++) {
+            imagePath = path.join(imagesPath, `${labelName}.${imageExts[0]}`);
+            imageName = `${labelName}.${imageExts[0]}`;
+            if (!fs.existsSync(imagePath)) {
+                imagePath = '';
+                continue;
+            }
+        }
+        // 判断图片是否存在
+        if (!imagePath) {
+            logStrArr.push(`[E] 图片不存在 ${filePath} ${fileData} 可能是图片后缀异常 [ ${imageExts.join(', ')} ]`);
+            continue;
+        }
+
+        readline.cursorTo(process.stdout, 0);
+        readline.clearScreenDown(process.stdout);
+        process.stdout.write(`${i} / ${labelFiles.length} | 解析文件: ${filePath} 中\n`);
+        // console.log(result);
+        // 空数据统计
+        if (result.length === 0) {
+            countMap['空'] = (countMap['空'] || 0) + 1;
+            // 复制图片文件到 emptyPath
+
+            let emptyFilePath = path.join(emptyPath, `${labelName}.${imageExts[0]}`);
+            logStrArr.push(`[E] ${labelName}内容为空 空文件 ${emptyFilePath}`);
+            console.log(`空文件: ${emptyFilePath}`)
+            cpFile(imagePath, emptyFilePath).then(_=>_)
+            continue;
+        }
+        // 获取文件标注类型
+        let fileType = {};
+        for (let j = 0; j < result.length; j++) {
+            // let type = item.type;
+            let typeName = dimensionType[result[j].type];
+            if (!typeName) {
+                console.log('未知类型')
+                logStrArr.push(`[E] 未知类型 ${filePath} ${fileData}`);
+                continue;
+            }
+            if (fileType[typeName]) {
+                fileType[typeName]++;
+            } else {
+                fileType[typeName] = 1;
+            }
+        }
+        console.log(fileType)
+        if (Object.keys(fileType).length === 0) {
+            countMap['异常标注'] = (countMap['异常标注'] || 0) + 1;
+            continue;
+        }
+
+        // 获取标注的所有类型, 将图片进行区分  类型1:数量  类型2:数量 类型1-类型2:数量
+        let fileTypeKeys = Object.keys(fileType);
+        // 排序
+        fileTypeKeys.sort();
+        // 数据转换
+        fileTypeKeys = fileTypeKeys.join('-');
+
+        // 转换类型
+        if (countMap[fileTypeKeys]) {
+            countMap[fileTypeKeys]++;
+        } else {
+            countMap[fileTypeKeys] = 1;
+        }
+
+
+
+        // 尝试复制文件
+        let transferFilePath = path.join(transferPath, fileTypeKeys);
+        let transferImagePath = path.join(transferFilePath, `images/${imageName}`  );
+        let transferLabelPath = path.join(transferFilePath, `labels/${labelFile}`);
+        // 文件转移
+        // 同步复制文件
+
+        cpFile(filePath, transferLabelPath).then(_=>_)
+        cpFile(imagePath, transferImagePath).then(_=>_)
+        logStrArr.push(`${fileTypeKeys}: ${countMap[fileTypeKeys]} ${filePath}  ${imagePath} ===>  ${transferLabelPath} ${transferImagePath} `);
+        console.log(logStrArr[logStrArr.length - 1]);
+    }
+
+    // 输出结果
+    console.log(countMap);
+    // 将分析结果写入文件
+    logStrArr.push(`\r\n${JSON.stringify(countMap, null, 4)}`);
+    resultStrArr.push(`${JSON.stringify(countMap, null, 4)}`);
+    // 保存结果
+    let logStr = logStrArr.join('\r\n');
+    let resultStr = resultStrArr.join('\r\n');
+    await writeFile(path.join(logsPath, 'result.txt'), resultStr);
+    await writeFile(path.join(logsPath, 'log.txt'), logStr);
+    console.log('end');
+}
+
+
+
+
+
+// 将新处理的labels 重新写回至原目录
+function reCopyToRaw(){
+    let labelsPath = `E:\\图库\\ai\\epower\\newLabels`
+    // 数据转移目录
+    let transferPath = `E:\\图库\\ai\\epower\\parse`
+        // "0": "吊车",
+        // "1": "塔吊",
+        // "2": "烟火",
+        // "3": "施工机械",
+        // "4": "导线异物",
+    // 吊车-导线异物
+    // 施工机械-导线异物
+    // 塔吊-导线异物
+    // 塔吊-施工机械
+
+    let fileTypeKeys = [1, 3]
+    fileTypeKeys = fileTypeKeys.map(item =>  dimensionType[item] ? dimensionType[item] : '未知类型');
+    let targetType = fileTypeKeys.join('-');
+    let targetPath = path.join(transferPath, targetType);
+    let targetLabelPath = path.join(targetPath, 'labels');
+    let logStrArr = [];
+    // 遍历目录
+    let labelFiles = fs.readdirSync(targetLabelPath);
+    for (let i = 0; i < labelFiles.length; i++) {
+        let labelFilePath = path.join(targetLabelPath, labelFiles[i]);
+        // 拷贝文件
+        cpFile(labelFilePath, path.join(labelsPath, labelFiles[i])).then(_=>_)
+    }
+    console.log(targetPath);
+}
+
+
+// 复制分类好的label文件至指定目录
+async function copyLabel() {
+    // let allLabelsPath = `E:\\图库\\ai\\epower_v2\\parse`
+    let allLabelsPath = `E:\\图库\\ai\\epower\\parse`
+    // let transferPath = `E:\\图库\\ai\\epower_v2\\tmpLabels_2`
+    let transferPath = `E:\\图库\\ai\\epower\\tmpLabels_2`
+    // 获取子目录
+    let subDirs = fs.readdirSync(allLabelsPath);
+    for (let i = 0; i < subDirs.length; i++) {
+
+        if (subDirs[i] == 'classes.txt'){
+            console.log(`skip classes.txt`)
+            continue;
+        }
+        let subDir = path.join(allLabelsPath, subDirs[i], 'labels');
+        console.log(subDir)
+        // 获取子目录下的labels
+        let labelFiles = fs.readdirSync(subDir);
+        for (let j = 0; j < labelFiles.length; j++) {
+            // 获取文件路径
+            let labelFilePath = path.join(subDir, labelFiles[j]);
+            // 复制文件 至新目录 新目录: 新目录/子目录名称
+            let newDir = path.join(transferPath, subDirs[i]);
+            let newFilePath = path.join(newDir, labelFiles[j]);
+
+            // 复制文件
+            await cpFile(labelFilePath, newFilePath)
+        }
+    }
+}
+
+// 将转移出来的label文件合并至一个文件夹内
+async function copyLabelToAll() {
+    let allLabelsPath = `E:\\图库\\ai\\epower\\tmpLabels_2`
+    let baseImagesPath = `E:\\图库\\ai\\epower\\images`
+    let transferPath = `E:\\图库\\ai\\epower\\all_tmpLabels_2`
+    let trabsferImaegsPath = `E:\\图库\\ai\\epower\\all_tmpImages_2`
+    let imageExts = ['jpg']
+
+    // 获取子目录
+    let subDirs = fs.readdirSync(allLabelsPath);
+    for (let i = 0; i < subDirs.length; i++) {
+
+        if (subDirs[i] == 'classes.txt'){
+            console.log(`skip classes.txt`)
+            continue;
+        }
+        let subDir = path.join(allLabelsPath, subDirs[i]);
+        console.log(subDir)
+        // 获取子目录下的labels
+        let labelFiles = fs.readdirSync(subDir);
+        for (let j = 0; j < labelFiles.length; j++) {
+            if (labelFiles[j] == 'classes.txt'){
+                console.log(`skip classes.txt`)
+                continue;
+            }
+            let labelName = labelFiles[j].replace('.txt', '');
+            // 获取文件路径
+            let labelFilePath = path.join(subDir, labelFiles[j]);
+            // 复制文件 至新目录 新目录: 新目录/子目录名称
+            let newFilePath = path.join(transferPath, labelFiles[j]);
+            // 复制文件
+            cpFile(labelFilePath, newFilePath)
+
+            // 复制图片
+            // 图片后缀可能为 jpg, png 等
+            let imagePath = '';
+            let imageName = `${labelName}.jpg`;
+            imagePath = path.join(baseImagesPath, imageName);
+
+            // 复制图片
+            let newImagePath = path.join(trabsferImaegsPath, imageName);
+            console.log(`移动图片 ${imagePath} => ${newImagePath}`)
+            // 判断图片是否存在
+            if (!imagePath) {
+                console.log(`[E] 图片不存在 ${labelFilePath} 可能是图片后缀异常 [ ${imageExts.join(', ')} ]`);
+                continue;
+            }
+            cpFile(imagePath, newImagePath)
+        }
+    }
+}
+
+function replaceNewlines(input) {
+    return input.replace(/(\r\n|\r|\n)/g, '\r\n');
+}
+// 将label 文件中的换行 从 \n 替换为 \r\n
+function relineLabel(){
+    let labelsPath = `E:\\图库\\ai\\epower\\all_tmpLabels_2`
+    // 遍历目录
+    let labelFiles = fs.readdirSync(labelsPath);
+    for (let i = 0; i < labelFiles.length; i++) {
+        let labelFilePath = path.join(labelsPath, labelFiles[i]);
+        if (!fs.statSync(labelFilePath).isFile()){
+            continue;
+        }
+        console.log(`start reline ${labelFilePath}`);
+        let fileContent = fs.readFileSync(labelFilePath, 'utf-8');
+        fileContent = replaceNewlines(fileContent);
+        fs.writeFileSync(labelFilePath, fileContent)
+    }
+}
+
+
+
+main();
+
+// reCopyToRaw();
+
+// copyLabel();
+
+// copyLabelToAll()
+
+// relineLabel();
+