Skip to content
This repository has been archived by the owner on May 10, 2023. It is now read-only.

Commit

Permalink
fix: migrate pa to pa-IN and remove CV -> SC language mapping
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaelKohler committed Jun 26, 2021
1 parent 418567d commit d9f5f04
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 15 deletions.
12 changes: 3 additions & 9 deletions scripts/exporter.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,6 @@ const cleanup = require('../server/lib/cleanup');
const CV_LANGUAGES_URL = 'https://raw.githubusercontent.com/mozilla/common-voice/main/locales/all.json';
const OUTPUT_TXT = 'sentence-collector.txt';

// Mapping from PONTOON locale -> SC locale code
const LANGUAGE_MAPPING = {
'pa-IN': 'pa',
};

const {
API_BASE_URL,
COMMON_VOICE_PATH,
Expand Down Expand Up @@ -48,9 +43,8 @@ async function startExport() {
async function exportLanguage(languageCode) {
console.log(`Starting export for ${languageCode}..`);

const dbLanguageCode = LANGUAGE_MAPPING[languageCode] || languageCode;
const cvPath = `${exportPath}/${languageCode}`;
const approvedSentencesUrl = `${API_BASE_URL}/sentences/text/approved/${dbLanguageCode}`;
const approvedSentencesUrl = `${API_BASE_URL}/sentences/text/approved/${languageCode}`;

const approvedSentencesResponse = await fetch(approvedSentencesUrl);
const approvedSentencesText = await approvedSentencesResponse.text();
Expand All @@ -64,8 +58,8 @@ async function exportLanguage(languageCode) {
prepareExport(cvPath);

console.log(` - Cleaning up sentences`);
const cleanedUpSentences = cleanup.cleanupSentences(dbLanguageCode, approvedSentences);
const dedupedSentences = dedupeSentences(dbLanguageCode, cleanedUpSentences, cvPath);
const cleanedUpSentences = cleanup.cleanupSentences(languageCode, approvedSentences);
const dedupedSentences = dedupeSentences(languageCode, cleanedUpSentences, cvPath);

writeExport(cvPath, dedupedSentences);
}
Expand Down
12 changes: 6 additions & 6 deletions server/lib/languages.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,8 @@ const ISO6391 = require('iso-639-1');

const FALLBACK_LOCALE = 'en';

const LANGUAGE_MAPPING = {
// CV - Sentence Collector
'pa-IN': 'pa',
};

module.exports = {
FALLBACK_LOCALE,
LANGUAGE_MAPPING,
getAllLanguages,
};

Expand Down Expand Up @@ -164,6 +158,11 @@ const ADDITIONAL_LANGUAGES = [
name: 'Norwegian',
nativeName: 'Norsk nynorsk',
},
{
id: 'pa-IN',
name: 'Panjabi',
nativeName: 'ਪੰਜਾਬੀ',
},
{ // https://github.com/common-voice/common-voice/issues/3044
id: 'pap-AW',
name: 'Papiamento - Aruba',
Expand Down Expand Up @@ -257,6 +256,7 @@ const LANGUAGES_TO_REMOVE = [
'ga', // covered by ga-IE
'sv', // covered by sv-SE
'ne', // covered by ne-NP
'pa', // covered by pa-IN
];

const isoLanguages = ISO6391.getLanguages(ISO6391.getAllCodes());
Expand Down
12 changes: 12 additions & 0 deletions server/migrations/20210626134900-migrate-pa-to-pa-IN.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
'use strict';

module.exports = {
up: (queryInterface) => {
return queryInterface.sequelize.query(`
UPDATE Sentences
SET localeId="pa-IN"
WHERE localeId="pa"
`);
},
down: () => Promise.resolve(),
};
11 changes: 11 additions & 0 deletions server/migrations/20210626134930-migrate-pa-users.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
'use strict';

module.exports = {
up: async (queryInterface) => {
await queryInterface.sequelize.query("UPDATE Users SET languages = REPLACE(languages, 'pa', 'pa-IN') WHERE languages='pa'");
await queryInterface.sequelize.query("UPDATE Users SET languages = REPLACE(languages, 'pa,', 'pa-IN,') WHERE languages LIKE 'pa,%'");
await queryInterface.sequelize.query("UPDATE Users SET languages = REPLACE(languages, ',pa,', ',pa-IN,') WHERE languages LIKE '%,pa,%'");
await queryInterface.sequelize.query("UPDATE Users SET languages = REPLACE(languages, ',pa', ',pa-IN') WHERE languages LIKE '%,pa'");
},
down: () => Promise.resolve(),
};

0 comments on commit d9f5f04

Please sign in to comment.