Skip to content

Commit

Permalink
Merge pull request #569 from Helium314/compress
Browse files Browse the repository at this point in the history
Compress dictionaries and allow users to add more
  • Loading branch information
MajeurAndroid authored Mar 21, 2022
2 parents c71acea + 40c09e4 commit 8d92f1a
Show file tree
Hide file tree
Showing 27 changed files with 255 additions and 11 deletions.
3 changes: 2 additions & 1 deletion app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ android {

ndkVersion '21.3.6528147'
androidResources {
noCompress 'dict'
noCompress 'main.dict'
noCompress 'empty.dict'
}
}

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import android.content.res.AssetFileDescriptor;
import android.util.Log;

import org.dslul.openboard.inputmethod.latin.common.FileUtils;
import org.dslul.openboard.inputmethod.latin.common.LocaleUtils;
import org.dslul.openboard.inputmethod.latin.define.DecoderSpecificConstants;
import org.dslul.openboard.inputmethod.latin.makedict.DictionaryHeader;
Expand Down Expand Up @@ -62,6 +63,8 @@ final public class BinaryDictionaryGetter {
public static final String MAIN_DICTIONARY_CATEGORY = "main";
public static final String ID_CATEGORY_SEPARATOR = ":";

public static final String ASSETS_DICTIONARY_FOLDER = "dicts";

// The key considered to read the version attribute in a dictionary file.
private static String VERSION_KEY = "version";

Expand Down Expand Up @@ -265,14 +268,83 @@ public static ArrayList<AssetFileAddress> getDictionaryFiles(final Locale locale
}

if (!foundMainDict && dictPackSettings.isWordListActive(mainDictId)) {
final int fallbackResId =
DictionaryInfoUtils.getMainDictionaryResourceId(context.getResources(), locale);
final AssetFileAddress fallbackAsset = loadFallbackResource(context, fallbackResId);
final File dict = loadDictionaryFromAssets(locale.toString(), context);
final AssetFileAddress fallbackAsset;
if (dict == null) {
// fall back to the old way (maybe remove? will not work if files are compressed)
final int fallbackResId =
DictionaryInfoUtils.getMainDictionaryResourceId(context.getResources(), locale);
fallbackAsset = loadFallbackResource(context, fallbackResId);
} else {
fallbackAsset = AssetFileAddress.makeFromFileName(dict.getPath());
}
if (null != fallbackAsset) {
fileList.add(fallbackAsset);
}
}

return fileList;
}

/**
* Returns the best matching main dictionary from assets.
*
* Actually copies the dictionary to cache folder, and then returns that file. This allows
* the dictionaries to be stored in a compressed way, reducing APK size.
* On next load, the dictionary in cache folder is found by getCachedWordLists
*
* Returns null on IO errors or if no matching dictionary is found
*/
public static File loadDictionaryFromAssets(final String locale, final Context context) {
final String[] dictionaryList;
try {
dictionaryList = context.getAssets().list(ASSETS_DICTIONARY_FOLDER);
} catch (IOException e) {
return null;
}
if (null == dictionaryList) return null;
String bestMatchName = null;
int bestMatchLevel = 0;
for (String dictionary : dictionaryList) {
final String dictLocale =
extractLocaleFromAssetsDictionaryFile(dictionary);
if (dictLocale == null) continue;
final int matchLevel = LocaleUtils.getMatchLevel(dictLocale, locale);
if (LocaleUtils.isMatch(matchLevel) && matchLevel > bestMatchLevel) {
bestMatchName = dictionary;
}
}
if (bestMatchName == null) return null;

// we have a match, now copy contents of the dictionary to "cached" word lists folder
File dictFile = new File(DictionaryInfoUtils.getCacheDirectoryForLocale(bestMatchName, context) +
File.separator + DictionaryInfoUtils.MAIN_DICTIONARY_INTERNAL_FILE_NAME);
try {
FileUtils.copyStreamToNewFile(
context.getAssets().open(ASSETS_DICTIONARY_FOLDER + File.separator + bestMatchName),
dictFile);
return dictFile;
} catch (IOException e) {
Log.e(TAG, "exception while looking for locale " + locale, e);
return null;
}
}

/**
* Returns the locale for a dictionary file name stored in assets.
*
* Assumes file name main_[locale].dict
*
* Returns the locale, or null if file name does not match the pattern
*/
private static String extractLocaleFromAssetsDictionaryFile(final String dictionaryFileName) {
if (dictionaryFileName.startsWith(BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY)
&& dictionaryFileName.endsWith(".dict")) {
return dictionaryFileName.substring(
BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY.length() + 1,
dictionaryFileName.lastIndexOf('.')
);
}
return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
package org.dslul.openboard.inputmethod.latin.common;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;

/**
* A simple class to help with removing directories recursively.
Expand Down Expand Up @@ -58,4 +61,19 @@ public static boolean renameTo(final File fromFile, final File toFile) {
toFile.delete();
return fromFile.renameTo(toFile);
}

public static void copyStreamToNewFile(InputStream in, File outfile) throws IOException {
File parentFile = outfile.getParentFile();
if (parentFile == null || (!parentFile.exists() && !parentFile.mkdirs())) {
throw new IOException("could not create parent folder");
}
FileOutputStream out = new FileOutputStream(outfile);
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) > 0) {
out.write(buf, 0, len);
}
out.flush();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,34 @@

package org.dslul.openboard.inputmethod.latin.settings;

import android.Manifest;
import android.app.Activity;
import android.app.AlertDialog;
import android.content.Context;
import android.content.DialogInterface;
import android.content.Intent;
import android.content.SharedPreferences;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.net.Uri;
import android.os.Bundle;
import android.preference.Preference;
import android.preference.SwitchPreference;
import android.text.TextUtils;
import android.text.Html;
import android.text.Spanned;
import android.text.method.LinkMovementMethod;
import android.widget.TextView;
import android.widget.Toast;

import org.dslul.openboard.inputmethod.dictionarypack.DictionaryPackConstants;
import org.dslul.openboard.inputmethod.latin.R;
import org.dslul.openboard.inputmethod.latin.permissions.PermissionsManager;
import org.dslul.openboard.inputmethod.latin.permissions.PermissionsUtil;
import org.dslul.openboard.inputmethod.latin.common.FileUtils;
import org.dslul.openboard.inputmethod.latin.makedict.DictionaryHeader;
import org.dslul.openboard.inputmethod.latin.userdictionary.UserDictionaryList;
import org.dslul.openboard.inputmethod.latin.userdictionary.UserDictionarySettings;
import org.dslul.openboard.inputmethod.latin.utils.DialogUtils;
import org.dslul.openboard.inputmethod.latin.utils.DictionaryInfoUtils;

import java.io.File;
import java.io.IOException;
import java.util.TreeSet;

/**
Expand All @@ -55,6 +65,8 @@ public final class CorrectionSettingsFragment extends SubScreenFragment
private static final boolean DBG_USE_INTERNAL_PERSONAL_DICTIONARY_SETTINGS = false;
private static final boolean USE_INTERNAL_PERSONAL_DICTIONARY_SETTINGS =
DBG_USE_INTERNAL_PERSONAL_DICTIONARY_SETTINGS;
private static final int DICTIONARY_REQUEST_CODE = 96834;
private static final String DICTIONARY_URL = "https://github.com/openboard-team/openboard/"; // TODO: update once it exists

@Override
public void onCreate(final Bundle icicle) {
Expand All @@ -73,6 +85,19 @@ public void onCreate(final Bundle icicle) {
if (ri == null) {
overwriteUserDictionaryPreference(editPersonalDictionary);
}

// Ideally this would go to a preference screen where extra dictionaries can be managed
// so user can check which dictionaries exists (internal and added), and also delete them.
// But for now just adding new ones and replacing is ok.
final Preference addDictionary = findPreference(Settings.PREF_ADD_DICTIONARY);
if (addDictionary != null)
addDictionary.setOnPreferenceClickListener(new Preference.OnPreferenceClickListener() {
@Override
public boolean onPreferenceClick(Preference preference) {
showAddDictionaryDialog();
return true;
}
});
}

private void overwriteUserDictionaryPreference(final Preference userDictionaryPreference) {
Expand All @@ -99,4 +124,111 @@ private void overwriteUserDictionaryPreference(final Preference userDictionaryPr
}
}

private void showAddDictionaryDialog() {
final String link = "<a href='" + DICTIONARY_URL + "'>" +
getResources().getString(R.string.dictionary_selection_link_text) + "</a>";
final Spanned message = Html.fromHtml(getResources().getString(R.string.dictionary_selection_message, link));
final AlertDialog dialog = new AlertDialog.Builder(
DialogUtils.getPlatformDialogThemeContext(getActivity()))
.setTitle(R.string.dictionary_selection_title)
.setMessage(message)
.setNegativeButton(R.string.cancel, null)
.setPositiveButton(R.string.dictionary_selection_load_file, new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog, int which) {
Intent intent = new Intent(Intent.ACTION_OPEN_DOCUMENT)
.addCategory(Intent.CATEGORY_OPENABLE)
.setType("application/octet-stream");
startActivityForResult(intent, DICTIONARY_REQUEST_CODE);
}
})
.create();
dialog.show();
// make links in the HTML text work
((TextView) dialog.findViewById(android.R.id.message))
.setMovementMethod(LinkMovementMethod.getInstance());
}

private void onDictionaryFileSelected(int resultCode, Intent resultData) {
if (resultCode != Activity.RESULT_OK || resultData == null) {
onDictionaryLoadingError(R.string.dictionary_selection_error);
return;
}

final Uri uri = resultData.getData();
if (uri == null) {
onDictionaryLoadingError(R.string.dictionary_selection_error);
return;
}

final File cachedDictionaryFile = new File(getActivity().getCacheDir().getPath() + File.separator + "temp_dict");
try {
FileUtils.copyStreamToNewFile(
getActivity().getContentResolver().openInputStream(uri),
cachedDictionaryFile);
} catch (IOException e) {
onDictionaryLoadingError(R.string.dictionary_selection_error);
return;
}

final DictionaryHeader newHeader = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(cachedDictionaryFile, 0, cachedDictionaryFile.length());
if (newHeader == null) {
cachedDictionaryFile.delete();
onDictionaryLoadingError(R.string.dictionary_selection_file_error);
return;
}

final String dictFolder =
DictionaryInfoUtils.getCacheDirectoryForLocale(newHeader.mLocaleString, getActivity());
final File dictFile = new File(dictFolder + File.separator + DictionaryInfoUtils.MAIN_DICTIONARY_USER_FILE_NAME);
if (dictFile.exists()) {
final DictionaryHeader oldHeader =
DictionaryInfoUtils.getDictionaryFileHeaderOrNull(dictFile, 0, dictFile.length());
if (oldHeader != null
&& Integer.parseInt(oldHeader.mVersionString) > Integer.parseInt(newHeader.mVersionString)
&& !shouldReplaceExistingUserDictionary()) {
cachedDictionaryFile.delete();
return;
}
}

if (!cachedDictionaryFile.renameTo(dictFile)) {
cachedDictionaryFile.delete();
onDictionaryLoadingError(R.string.dictionary_selection_error);
return;
}

// success, now remove internal dictionary file if it exists
final File internalDictFile = new File(dictFolder + File.separator +
DictionaryInfoUtils.MAIN_DICTIONARY_INTERNAL_FILE_NAME);
if (internalDictFile.exists())
internalDictFile.delete();

// inform user about success
final String successMessageForLocale = getResources()
.getString(R.string.dictionary_selection_load_success, newHeader.mLocaleString);
Toast.makeText(getActivity(), successMessageForLocale, Toast.LENGTH_SHORT).show();

// inform LatinIME about new dictionary
final Intent newDictBroadcast = new Intent(DictionaryPackConstants.NEW_DICTIONARY_INTENT_ACTION);
getActivity().sendBroadcast(newDictBroadcast);
}

private void onDictionaryLoadingError(int resId) {
// show error message... maybe better as dialog so user definitely notices?
Toast.makeText(getActivity(), resId, Toast.LENGTH_LONG).show();
}

private boolean shouldReplaceExistingUserDictionary() {
// TODO: show dialog, ask user whether existing file should be replaced
// return true if yes, no otherwise (set .setCancelable(false) to avoid dismissing without the buttons!)
return true;
}

@Override
public void onActivityResult(int requestCode, int resultCode, Intent resultData) {
if (requestCode == DICTIONARY_REQUEST_CODE)
onDictionaryFileSelected(resultCode, resultData);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang
public static final String PREF_VOICE_INPUT_KEY = "pref_voice_input_key";
public static final String PREF_CLIPBOARD_CLIPBOARD_KEY = "pref_clipboard_clipboard_key";
public static final String PREF_EDIT_PERSONAL_DICTIONARY = "edit_personal_dictionary";
public static final String PREF_ADD_DICTIONARY = "add_dictionary";
// PREF_AUTO_CORRECTION_THRESHOLD_OBSOLETE is obsolete. Use PREF_AUTO_CORRECTION instead.
public static final String PREF_AUTO_CORRECTION_THRESHOLD_OBSOLETE =
"auto_correction_threshold";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ public class DictionaryInfoUtils {
public static final String RESOURCE_PACKAGE_NAME = R.class.getPackage().getName();
private static final String DEFAULT_MAIN_DICT = "main";
private static final String MAIN_DICT_PREFIX = "main_";
public static final String MAIN_DICTIONARY_INTERNAL_FILE_NAME = DEFAULT_MAIN_DICT + ".dict";
public static final String MAIN_DICTIONARY_USER_FILE_NAME = MAIN_DICT_PREFIX + "user.dict";
private static final String DECODER_DICT_SUFFIX = DecoderSpecificConstants.DECODER_DICT_SUFFIX;
// 6 digits - unicode is limited to 21 bits
private static final int MAX_HEX_DIGITS_FOR_CODEPOINT = 6;
Expand Down Expand Up @@ -151,7 +153,7 @@ public static String replaceFileNameDangerousCharacters(final String name) {
/**
* Helper method to get the top level cache directory.
*/
private static String getWordListCacheDirectory(final Context context) {
public static String getWordListCacheDirectory(final Context context) {
return context.getFilesDir() + File.separator + "dicts";
}

Expand Down Expand Up @@ -242,7 +244,7 @@ public static boolean isMainWordListId(final String id) {
// An id is supposed to be in format category:locale, so splitting on the separator
// should yield a 2-elements array
if (2 != idArray.length) {
return false;
return id.startsWith(BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY);
}
return BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY.equals(idArray[0]);
}
Expand Down
14 changes: 14 additions & 0 deletions app/src/main/res/values/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,20 @@ disposition rather than other common dispositions for Latin languages. [CHAR LIM
<string name="dictionary_settings_title">Add-on dictionaries</string>
<!-- Title for the prompt dialog which informs the user that a dictionary is available for the current language and asks to decide whether to download it over 3g -->
<string name="dictionary_settings_summary">Settings for dictionaries</string>
<!-- Title for the user dictionary selection dialog -->
<string name="dictionary_selection_title">"Choose dictionary file"</string>
<!-- Message for the user dictionary selection dialog. This string will be interpreted as HTML -->
<string name="dictionary_selection_message">"Select a dictionary to replace the main dictionary of the same locale. Dictionaries can be downloaded at %s."</string>
<!-- Title of the link to the download page inserted into selection message -->
<string name="dictionary_selection_link_text">"the project repository"</string>
<!-- Button text for dictionary file selection -->
<string name="dictionary_selection_load_file">"Load dictionary"</string>
<!-- Toast text shown when dictionary file was added successfully -->
<string name="dictionary_selection_load_success">"Dictionary for locale \"%s\" added"</string>
<!-- Text shown when dictionary file could not be read -->
<string name="dictionary_selection_file_error">"Error: Selected file is not a valid dictionary file"</string>
<!-- Text shown on other errors when loading dictionary file -->
<string name="dictionary_selection_error">"Error loading dictionary file"</string>
<!-- Name of the user dictionaries settings category -->
<string name="user_dictionaries">User dictionaries</string>
<!-- Name for the "user dictionary" preference item when there is only one -->
Expand Down
4 changes: 4 additions & 0 deletions app/src/main/res/xml/prefs_screen_correction.xml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
<intent android:action="android.settings.USER_DICTIONARY_SETTINGS" />
</PreferenceScreen>

<Preference
android:key="add_dictionary"
android:title="@string/configure_dictionaries_title" />

<PreferenceCategory
android:title="@string/settings_category_correction">

Expand Down

0 comments on commit 8d92f1a

Please sign in to comment.