[2.4.0 stack 3/18] Fixes across existing utils

scripts/cities.ts

@@ -4,10 +4,13 @@ import { writeFile } from "node:fs/promises";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 
+import { DATA as STATES } from "../src/_internals/constants/states.ts";
 import { fetchWithRetry } from "../src/_internals/fetch-with-retry/fetch-with-retry.ts";
 
 const scriptsDir = dirname(fileURLToPath(import.meta.url));
 
+const STATE_CODES = STATES.map((state) => state.code);
+
 type City = {
 	id: number;
 	nome: string;
@@ -28,6 +31,19 @@ type City = {
 				};
 			};
 		};
+	} | null;
+	"regiao-imediata": {
+		id: number;
+		nome: string;
+		"regiao-intermediaria": {
+			id: number;
+			nome: string;
+			UF: {
+				id: number;
+				sigla: string;
+				nome: string;
+			};
+		};
 	};
 };
 
@@ -40,57 +56,71 @@ const main = async () => {
 		throw new Error(`IBGE municipalities request failed with status ${response.status}`);
 	}
 
-	const json = (await response.json()) as City[];
+	const json: City[] = await response.json();
 
-	const cities = Object.fromEntries(
-		Object.entries(
-			json.reduce(
-				(acc, city) => {
-					const stateInitials = city?.microrregiao?.mesorregiao?.UF?.sigla;
+	const byState = json.reduce(
+		(acc, city) => {
+			const stateInitials =
+				city?.microrregiao?.mesorregiao?.UF?.sigla ??
+				city?.["regiao-imediata"]?.["regiao-intermediaria"]?.UF?.sigla;
 
-					if (!stateInitials) return acc;
+			if (!stateInitials) return acc;
 
-					if (!acc[stateInitials]) {
-						acc[stateInitials] = [];
-					}
+			if (!acc[stateInitials]) {
+				acc[stateInitials] = [];
+			}
 
-					acc[stateInitials].push(city.nome);
+			acc[stateInitials].push([city.nome, String(city.id)]);
 
-					return acc;
-				},
-				{} as Record<string, string[]>,
-			),
-		)
-			.sort(([a], [b]) => a.localeCompare(b))
-			.map(([state, cityNames]) => [state, cityNames.sort((a, b) => a.localeCompare(b))]),
+			return acc;
+		},
+		{} as Record<string, [string, string][]>,
 	);
 
+	const sortedEntries = Object.entries(byState)
+		.sort(([a], [b]) => a.localeCompare(b))
+		.map(
+			([state, tuples]) => [state, tuples.sort(([a], [b]) => a.localeCompare(b, "pt-BR"))] as const,
+		);
+
+	const body = sortedEntries
+		.map(([state, tuples]) => {
+			const rows = tuples.map(([name, code]) => `\t\t[${JSON.stringify(name)}, "${code}"],`);
+
+			return `\t${state}: [\n${rows.join("\n")}\n\t],`;
+		})
+		.join("\n");
+
+	const missingStates = STATE_CODES.filter((code) => !byState[code]);
+
+	if (missingStates.length > 0) {
+		throw new Error(`IBGE response is missing municipalities for: ${missingStates.join(", ")}`);
+	}
+
 	await writeFile(
 		resolve(scriptsDir, "..", "./src/_internals/constants/cities.ts"),
-		`/**
- * A collection of Brazilian cities categorized by their respective states.
- * 
- * @constant
- * @type {Object}
- * @property {string[]} GO - Cities in the state of Goiás.
- * @property {string[]} MG - Cities in the state of Minas Gerais.
- * @property {string[]} PA - Cities in the state of Pará.
- * @property {string[]} CE - Cities in the state of Ceará.
- * @property {string[]} BA - Cities in the state of Bahia.
- * @property {string[]} PR - Cities in the state of Paraná.
- * @property {string[]} SC - Cities in the state of Santa Catarina.
- * @property {string[]} PE - Cities in the state of Pernambuco.
- * @property {string[]} TO - Cities in the state of Tocantins.
- * @property {string[]} RN - Cities in the state of Rio Grande do Norte.
- * @property {string[]} PI - Cities in the state of Piauí.
- * @property {string[]} RS - Cities in the state of Rio Grande do Sul.
- * @property {string[]} MT - Cities in the state of Mato Grosso.
- * @property {string[]} AC - Cities in the state of Acre.
- * @property {string[]} SP - Cities in the state of São Paulo.
- * @property {string[]} ES - Cities in the state of Espírito Santo.
- * @property {string[]} MA - Cities in the state of Maranhão.
+		`import type { StateCode } from "./states";
+
+/**
+ * Brazilian municipalities by state, published by the IBGE. \`DATA\` holds, for each state, a
+ * \`[name, ibgeCode]\` tuple per municipality, sorted by name with \`localeCompare\` in the
+ * "pt-BR" locale. Generated by \`scripts/cities.ts\`.
+ *
+ * @see https://servicodados.ibge.gov.br/api/docs/localidades
  */
-export const DATA = ${JSON.stringify(cities)} as const`,
+export type Municipality = {
+	/** The 7-digit IBGE municipality code. */
+	code: string;
+	/** The municipality name. */
+	name: string;
+	/** The two-letter code of the state the municipality belongs to. */
+	stateCode: StateCode;
+};
+
+export const DATA: Record<StateCode, readonly (readonly [string, string])[]> = {
+${body}
+};
+`,
 	);
 };
 

scripts/legal-natures.ts

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+
+import { writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { inflateSync } from "node:zlib";
+
+import { fetchWithRetry } from "../src/_internals/fetch-with-retry/fetch-with-retry.ts";
+
+const scriptsDir = dirname(fileURLToPath(import.meta.url));
+
+const SOURCE_URL =
+	"https://concla.ibge.gov.br/images/concla/documentacao/CONCLA-TNJ2021-EstruturaDetalhada.pdf";
+
+const SOURCE_PAGE_URL =
+	"https://concla.ibge.gov.br/estrutura/natjur-estrutura/natureza-juridica-2021";
+
+const OUTPUT_PATH = "./src/is-valid-legal-nature/constants.ts";
+
+const EXPECTED_CODES = 92;
+
+const LEGACY_LEGAL_NATURE: Record<string, string> = {
+	"2076": "Sociedade Empresária em Nome Coletivo",
+	"2100": "Sociedade Mercantil de Capital e Indústria (extinta pelo NCC/2002)",
+	"2208": "Entidade Binacional Itaipu",
+	"3042": "Organização Social",
+	"3050": "Organização da Sociedade Civil de Interesse Público (Oscip)",
+	"3093": "Unidade Executora (Programa Dinheiro Direto na Escola)",
+	"3123": "Partido Político",
+	"5002": "Organização Internacional e Outras Instituições Extraterritoriais",
+};
+
+const TYPO_FIXES: Record<string, string> = {
+	"Frente Plebiscitária ou Referendaria": "Frente Plebiscitária ou Referendária",
+};
+
+const inflateStreams = (pdf: Buffer): string[] => {
+	const streams: string[] = [];
+	let cursor = 0;
+
+	while (cursor < pdf.length) {
+		const start = pdf.indexOf("stream", cursor);
+		if (start < 0) break;
+
+		let contentStart = start + "stream".length;
+		if (pdf[contentStart] === 0x0d) contentStart += 1;
+		if (pdf[contentStart] === 0x0a) contentStart += 1;
+
+		const end = pdf.indexOf("endstream", contentStart);
+		if (end < 0) break;
+
+		try {
+			streams.push(inflateSync(pdf.subarray(contentStart, end)).toString("latin1"));
+		} catch {}
+
+		cursor = end + "endstream".length;
+	}
+
+	return streams;
+};
+
+const unescapePdfString = (value: string): string =>
+	value.replace(/\\([0-7]{1,3})|\\(.)/g, (_match, octal?: string, char?: string) => {
+		if (octal) return String.fromCharCode(Number.parseInt(octal, 8));
+		if (char === "n") return "\n";
+		if (char === "r") return "\r";
+		if (char === "t") return "\t";
+		return char ?? "";
+	});
+
+const extractLines = (streams: string[]): string[] => {
+	const lines: string[] = [];
+
+	for (const stream of streams) {
+		const rows = new Map<number, Array<[number, string]>>();
+
+		for (const block of stream.matchAll(/BT([\s\S]*?)ET/g)) {
+			const body = block[1];
+			const matrix = [...body.matchAll(/([-\d.]+)\s+([-\d.]+)\s+Tm/g)].pop();
+			if (!matrix) continue;
+
+			let text = "";
+			for (const array of body.matchAll(/\[((?:[^[\]\\]|\\.)*)\]\s*TJ/g)) {
+				for (const chunk of array[1].matchAll(/\(((?:[^()\\]|\\.)*)\)/g)) {
+					text += unescapePdfString(chunk[1]);
+				}
+			}
+			for (const chunk of body.matchAll(/\(((?:[^()\\]|\\.)*)\)\s*Tj/g)) {
+				text += unescapePdfString(chunk[1]);
+			}
+			if (!text) continue;
+
+			const x = Number.parseFloat(matrix[1]);
+			const y = Math.round(Number.parseFloat(matrix[2]) * 10) / 10;
+
+			const row = rows.get(y) ?? [];
+			row.push([x, text]);
+			rows.set(y, row);
+		}
+
+		for (const y of [...rows.keys()].sort((a, b) => b - a)) {
+			const row = rows.get(y) ?? [];
+			lines.push(
+				row
+					.sort(([a], [b]) => a - b)
+					.map(([, text]) => text)
+					.join(""),
+			);
+		}
+	}
+
+	return lines;
+};
+
+const parseLegalNatures = (lines: string[]): Record<string, string> => {
+	const legalNatures: Record<string, string> = {};
+
+	for (const line of lines) {
+		const match = line.match(/^\s*(\d{3})-(\d)\s*-\s*(.+?)\s*$/);
+		if (!match) continue;
+
+		const code = `${match[1]}${match[2]}`;
+		const description = match[3].replace(/\s+/g, " ").trim();
+
+		legalNatures[code] = TYPO_FIXES[description] ?? description;
+	}
+
+	return legalNatures;
+};
+
+const stringifyEntries = (entries: Record<string, string>): string =>
+	Object.entries(entries)
+		.map(([code, description]) => `\t${JSON.stringify(code)}: ${JSON.stringify(description)},`)
+		.join("\n");
+
+const main = async () => {
+	const response = await fetchWithRetry(SOURCE_URL);
+
+	if (!response.ok) {
+		throw new Error(`CONCLA legal natures request failed with status ${response.status}`);
+	}
+
+	const pdf = Buffer.from(await response.arrayBuffer());
+	const current = parseLegalNatures(extractLines(inflateStreams(pdf)));
+	const codes = Object.keys(current);
+
+	if (codes.length !== EXPECTED_CODES) {
+		throw new Error(`Expected ${EXPECTED_CODES} legal natures, got ${codes.length}`);
+	}
+
+	const legacy = Object.fromEntries(
+		Object.entries(LEGACY_LEGAL_NATURE).filter(([code]) => !(code in current)),
+	);
+
+	await writeFile(
+		resolve(scriptsDir, "..", OUTPUT_PATH),
+		`/**
+ * Tabela de Natureza Jurídica 2021 (IBGE/CONCLA), indexed by the four digit code.
+ *
+ * Generated by \`node ./scripts/legal-natures.ts\`. Do not edit by hand.
+ *
+ * @see ${SOURCE_PAGE_URL}
+ * @see ${SOURCE_URL}
+ */
+export const LEGAL_NATURE: Record<string, string> = {
+${stringifyEntries(current)}
+
+${stringifyEntries(legacy)}
+};
+`,
+	);
+};
+
+await main().catch((error) => {
+	console.error(error instanceof Error ? error.message : error);
+	process.exit(1);
+});

scripts/states.ts

@@ -19,6 +19,23 @@ type State = {
 	};
 };
 
+const isState = (value: unknown): value is State =>
+	typeof value === "object" &&
+	value !== null &&
+	"id" in value &&
+	typeof value.id === "number" &&
+	"sigla" in value &&
+	typeof value.sigla === "string" &&
+	"nome" in value &&
+	typeof value.nome === "string" &&
+	"regiao" in value &&
+	typeof value.regiao === "object" &&
+	value.regiao !== null &&
+	"sigla" in value.regiao &&
+	typeof value.regiao.sigla === "string" &&
+	"nome" in value.regiao &&
+	typeof value.regiao.nome === "string";
+
 const main = async () => {
 	const response = await fetchWithRetry(
 		"https://servicodados.ibge.gov.br/api/v1/localidades/estados",
@@ -28,21 +45,32 @@ const main = async () => {
 		throw new Error(`IBGE states request failed with status ${response.status}`);
 	}
 
-	const json = (await response.json()) as State[];
+	const json: unknown = await response.json();
+
+	if (!Array.isArray(json) || json.length === 0 || !json.every(isState)) {
+		throw new Error(
+			"IBGE states payload is not an array of states with id, sigla, nome and regiao",
+		);
+	}
 
 	const states = json
-		.sort((cityA, cityB) => (cityA.nome > cityB.nome ? 1 : -1))
+		.sort((stateA, stateB) => stateA.nome.localeCompare(stateB.nome, "pt-BR"))
 		.map((state) => ({
 			code: state.sigla,
 			name: state.nome,
 			regionCode: state.regiao.sigla,
 			regionName: state.regiao.nome,
+			ibgeCode: state.id,
 		}));
 
 	await writeFile(
 		resolve(scriptsDir, "..", "./src/_internals/constants/states.ts"),
 		`/**
- * @type {Array<{code: string, name: string, regionCode: string, regionName: string}>}
+ * Brazilian states published by the IBGE, sorted by name with \`localeCompare\` in the "pt-BR"
+ * locale. \`ibgeCode\` is the 2-digit IBGE code of the Federative Unit ("cUF"), the same code
+ * found in the first field of every DF-e access key (chave de acesso).
+ *
+ * @see https://servicodados.ibge.gov.br/api/docs/localidades
  */
 export const DATA = ${JSON.stringify(states)} as const