feat: optimize UI queries and seed number library
This commit is contained in:
hectorzhao
@@ -0,0 +1,184 @@
|
||||
import fs from 'node:fs';
|
||||
import zlib from 'node:zlib';
|
||||
import crypto from 'node:crypto';
|
||||
import { PrismaClient } from '../packages/database/dist/index.js';
|
||||
|
||||
const args = Object.fromEntries(process.argv.slice(2).map((entry) => {
|
||||
const [key, ...value] = entry.replace(/^--/, '').split('=');
|
||||
return [key, value.length ? value.join('=') : true];
|
||||
}));
|
||||
|
||||
if (!args.admin || !args.phone) {
|
||||
throw new Error('Usage: node scripts/import-number-library.mjs --admin=/path/area.json.gz --phone=/path/phone.dat [--dry-run]');
|
||||
}
|
||||
|
||||
const batchId = String(args.batch || 'number-library-20260827');
|
||||
const adminSource = '国家统计局区划代码整理库 2024 (2023-06-30)';
|
||||
const phoneSource = 'pangongzi/phone phone.dat 2025-02';
|
||||
const adminTree = JSON.parse(zlib.gunzipSync(fs.readFileSync(String(args.admin))).toString('utf8'));
|
||||
|
||||
const sixDigitCode = (code) => String(code).slice(0, 6);
|
||||
const normalize = (value) => String(value || '')
|
||||
.replace(/\s+/g, '')
|
||||
.replace(/特别行政区|维吾尔自治区|壮族自治区|回族自治区|自治区|土家族苗族自治州|苗族侗族自治州|藏族自治州|蒙古族藏族自治州|柯尔克孜自治州|哈萨克自治州|自治州|省|市|地区|盟|县$/g, '');
|
||||
|
||||
const cities = [];
|
||||
for (const province of adminTree) {
|
||||
for (const city of province.children || []) {
|
||||
const base = {
|
||||
code: sixDigitCode(city.code),
|
||||
provinceCode: sixDigitCode(province.code),
|
||||
provinceName: province.name,
|
||||
cityCode: sixDigitCode(city.code),
|
||||
cityName: city.name === '市辖区' ? province.name : city.name,
|
||||
cityLevel: 'PREFECTURE',
|
||||
status: 'ENABLED'
|
||||
};
|
||||
cities.push(base);
|
||||
if (/直辖县级行政区划/.test(city.name)) {
|
||||
cities.pop();
|
||||
for (const countyCity of city.children || []) {
|
||||
cities.push({ ...base, code: sixDigitCode(countyCity.code), cityCode: sixDigitCode(countyCity.code), cityName: countyCity.name, cityLevel: 'COUNTY_DIRECT' });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const phoneBuffer = fs.readFileSync(String(args.phone));
|
||||
const indexOffset = phoneBuffer.readUInt32LE(4);
|
||||
const phoneRecords = [];
|
||||
for (let offset = indexOffset; offset + 9 <= phoneBuffer.length; offset += 9) {
|
||||
const segment7 = String(phoneBuffer.readUInt32LE(offset)).padStart(7, '0');
|
||||
const dataOffset = phoneBuffer.readUInt32LE(offset + 4);
|
||||
const end = phoneBuffer.indexOf(0, dataOffset);
|
||||
const [provinceName, cityName, zipCode, areaCode] = phoneBuffer.subarray(dataOffset, end).toString('utf8').split('|');
|
||||
phoneRecords.push({ segment7, provinceName, cityName, zipCode, areaCode, type: phoneBuffer[offset + 8] });
|
||||
}
|
||||
|
||||
const byProvince = new Map();
|
||||
for (const city of cities) {
|
||||
const key = normalize(city.provinceName);
|
||||
const values = byProvince.get(key) || [];
|
||||
values.push(city);
|
||||
byProvince.set(key, values);
|
||||
}
|
||||
|
||||
function findCity(record) {
|
||||
const provinceCities = byProvince.get(normalize(record.provinceName)) || [];
|
||||
const aliasKey = `${normalize(record.provinceName)}|${normalize(record.cityName)}`;
|
||||
const cityAliases = {
|
||||
'山东|莱芜': '济南',
|
||||
'新疆|巴州': '巴音郭楞',
|
||||
'新疆|博州': '博尔塔拉',
|
||||
'新疆|克州': '克孜勒苏',
|
||||
'新疆|奎屯': '伊犁',
|
||||
'青海|格尔木': '海西'
|
||||
};
|
||||
const cityName = cityAliases[aliasKey] || normalize(record.cityName || record.provinceName);
|
||||
const exact = provinceCities.find((city) => normalize(city.cityName) === cityName);
|
||||
if (exact) return exact;
|
||||
const partial = provinceCities.filter((city) => normalize(city.cityName).startsWith(cityName) || cityName.startsWith(normalize(city.cityName)));
|
||||
return partial.length === 1 ? partial[0] : provinceCities.length === 1 ? provinceCities[0] : null;
|
||||
}
|
||||
|
||||
const directMatches = phoneRecords.map((record) => ({ record, city: findCity(record) }));
|
||||
const areaVotes = new Map();
|
||||
for (const { record, city } of directMatches) {
|
||||
if (!city || !record.areaCode) continue;
|
||||
const key = `${record.areaCode}|${city.cityCode}`;
|
||||
areaVotes.set(key, (areaVotes.get(key) || 0) + 1);
|
||||
}
|
||||
const areaCity = new Map();
|
||||
for (const [key, count] of areaVotes) {
|
||||
const [areaCode, cityCode] = key.split('|');
|
||||
const current = areaCity.get(areaCode);
|
||||
if (!current || count > current.count) areaCity.set(areaCode, { cityCode, count });
|
||||
}
|
||||
const cityByCode = new Map(cities.map((city) => [city.cityCode, city]));
|
||||
const unmatched = [];
|
||||
const carrierMap = { 1: 'MOBILE', 2: 'UNICOM', 3: 'TELECOM', 4: 'MVNO', 5: 'MVNO', 6: 'MVNO', 7: 'BROADCAST', 8: 'MVNO' };
|
||||
const segments = [];
|
||||
for (const match of directMatches) {
|
||||
if (!/^1\d{6}$/.test(match.record.segment7)) continue;
|
||||
const city = match.city || cityByCode.get(areaCity.get(match.record.areaCode)?.cityCode);
|
||||
if (!city) {
|
||||
unmatched.push(match.record);
|
||||
continue;
|
||||
}
|
||||
segments.push({
|
||||
segment7: match.record.segment7,
|
||||
cityCode: city.cityCode,
|
||||
provinceName: city.provinceName,
|
||||
cityName: city.cityName,
|
||||
carrier: carrierMap[match.record.type] || 'UNKNOWN',
|
||||
source: phoneSource,
|
||||
batchId
|
||||
});
|
||||
}
|
||||
|
||||
const areaCodes = [...areaCity.entries()].map(([areaCode, vote]) => {
|
||||
const city = cityByCode.get(vote.cityCode);
|
||||
return { areaCode, cityCode: city.cityCode, provinceName: city.provinceName, cityName: city.cityName, source: phoneSource, batchId };
|
||||
}).filter((item) => /^0\d{2,3}$/.test(item.areaCode));
|
||||
|
||||
const prefixVotes = new Map();
|
||||
for (const segment of segments) {
|
||||
const prefix = segment.segment7.slice(0, 3);
|
||||
const key = `${prefix}|${segment.carrier}`;
|
||||
prefixVotes.set(key, (prefixVotes.get(key) || 0) + 1);
|
||||
}
|
||||
const prefixWinners = new Map();
|
||||
for (const [key, count] of prefixVotes) {
|
||||
const [prefix, carrier] = key.split('|');
|
||||
const current = prefixWinners.get(prefix);
|
||||
if (!current || count > current.count) prefixWinners.set(prefix, { carrier, count });
|
||||
}
|
||||
const prefixes = [...prefixWinners.entries()].map(([prefix, value]) => ({ prefix, carrier: value.carrier, priority: 100, source: phoneSource, batchId }));
|
||||
|
||||
const report = {
|
||||
batchId,
|
||||
adminSource,
|
||||
phoneSource,
|
||||
phoneVersion: phoneBuffer.subarray(0, 4).toString('utf8'),
|
||||
cities: cities.length,
|
||||
sourcePhoneRecords: phoneRecords.length,
|
||||
segments: segments.length,
|
||||
unmatched: unmatched.length,
|
||||
areaCodes: areaCodes.length,
|
||||
prefixes: prefixes.length,
|
||||
unmatchedLocations: [...new Map(unmatched.map(({ provinceName, cityName, areaCode }) => [`${provinceName}|${cityName}|${areaCode}`, { provinceName, cityName, areaCode }])).values()]
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
|
||||
if (args['dry-run']) process.exit(0);
|
||||
if (unmatched.length) throw new Error(`Refusing partial import: ${unmatched.length} phone segments could not be mapped to an administrative city.`);
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
try {
|
||||
const current = {
|
||||
cities: await prisma.geoCity.count(),
|
||||
segments: await prisma.phoneNumberSegment.count(),
|
||||
areaCodes: await prisma.phoneAreaCode.count(),
|
||||
prefixes: await prisma.carrierPrefixRule.count()
|
||||
};
|
||||
if (Object.values(current).some(Boolean)) throw new Error(`Number library is not empty: ${JSON.stringify(current)}`);
|
||||
|
||||
await prisma.geoCity.createMany({ data: cities, skipDuplicates: true });
|
||||
for (let offset = 0; offset < segments.length; offset += 1000) {
|
||||
await prisma.phoneNumberSegment.createMany({ data: segments.slice(offset, offset + 1000), skipDuplicates: true });
|
||||
}
|
||||
await prisma.phoneAreaCode.createMany({ data: areaCodes, skipDuplicates: true });
|
||||
await prisma.carrierPrefixRule.createMany({ data: prefixes, skipDuplicates: true });
|
||||
await prisma.outboxEvent.create({
|
||||
data: {
|
||||
id: `out_${crypto.randomUUID().replaceAll('-', '').slice(0, 28)}`,
|
||||
aggregateType: 'number_library_config',
|
||||
aggregateId: 'number-library',
|
||||
eventType: 'number_library.seeded',
|
||||
payload: report
|
||||
}
|
||||
});
|
||||
console.log(JSON.stringify({ imported: report }, null, 2));
|
||||
} finally {
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
Reference in New Issue
Block a user