Implement writing empty ver4 dictionary to file.
Bug: 11073222
Change-Id: Ic1a9295953d091c8e8982264ffff15669c30544c
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index b61a66c..e770d98 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -91,6 +91,8 @@
dynamic_patricia_trie_writing_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/, \
content/bigram_dict_content.cpp \
+ content/sparse_table_dict_content.cpp \
+ ver4_dict_buffers.cpp \
ver4_dict_constants.cpp \
ver4_patricia_trie_node_reader.cpp \
ver4_patricia_trie_node_writer.cpp \
@@ -100,6 +102,7 @@
buffer_with_extendable_buffer.cpp \
byte_array_utils.cpp \
dict_file_writing_utils.cpp \
+ file_utils.cpp \
forgetting_curve_utils.cpp \
format_utils.cpp \
mmapped_buffer.cpp \
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
index bc9e4b6..c5410b8 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
@@ -59,6 +59,12 @@
bool copyBigramList(const int bigramListPos, const int toPos);
+ bool flushToFile(const char *const dictDirPath) const {
+ return flush(dictDirPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
+ Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
+ Ver4DictConstants::BIGRAM_FILE_EXTENSION);
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
index c109cbf..5183764 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
@@ -62,6 +62,10 @@
Ver4DictConstants::PROBABILITY_SIZE, &probabilityWritingPos);
}
+ bool flushToFile(const char *const dictDirPath) const {
+ return flush(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(ProbabilityDictContent);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
index 8463a17..b11e233 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
@@ -60,6 +60,12 @@
return addressLookupTable->get(terminalId);
}
+ bool flushToFile(const char *const dictDirPath) const {
+ return flush(dictDirPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
+ Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
+ Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(ShortcutDictContent);
};
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
index 7669c1e..0878099 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
@@ -21,6 +21,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
@@ -54,6 +55,12 @@
return &mExpandableContentBuffer;
}
+ bool flush(const char *const dictDirPath, const char *const contentFileName) const {
+ const BufferWithExtendableBuffer *bufferPtr = &mExpandableContentBuffer;
+ return DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName,
+ &bufferPtr, 1 /* bufferCount */);
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(SingleDictContent);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
new file mode 100644
index 0000000..c654206
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
+
+namespace latinime {
+
+bool SparseTableDictContent::flush(const char *const dictDirPath,
+ const char *const lookupTableFileName, const char *const addressTableFileName,
+ const char *const contentFileName) const {
+ const BufferWithExtendableBuffer *lookupTableBufferPtr = &mExpandableLookupTableBuffer;
+ if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, lookupTableFileName,
+ &lookupTableBufferPtr, 1 /* bufferCount */)) {
+ return false;
+ }
+ const BufferWithExtendableBuffer *addressTableBufferPtr = &mExpandableAddressTableBuffer;
+ if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, addressTableFileName,
+ &addressTableBufferPtr, 1 /* bufferCount */)) {
+ return false;
+ }
+ const BufferWithExtendableBuffer *contentBufferPtr = &mExpandableContentBuffer;
+ if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName,
+ &contentBufferPtr, 1 /* bufferCount */)) {
+ return false;
+ }
+ return true;
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
index 5ae5f0f..bcfecdb 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
@@ -21,6 +21,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
@@ -85,6 +86,9 @@
return &mExpandableContentBuffer;
}
+ bool flush(const char *const dictDirPath, const char *const lookupTableFileName,
+ const char *const addressTableFileName, const char *const contentFileName) const;
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTableDictContent);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
index e016a2b..eaf18b5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
@@ -67,6 +67,10 @@
return mSize;
}
+ bool flushToFile(const char *const dictDirPath) const {
+ return flush(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(TerminalPositionLookupTable);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
new file mode 100644
index 0000000..e17c5ea
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
+
+#include <cerrno>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
+#include "suggest/policyimpl/dictionary/utils/file_utils.h"
+
+namespace latinime {
+
+bool Ver4DictBuffers::flush(const char *const dictDirPath) const {
+ // Create temporary directory.
+ const int tmpDirPathBufSize = FileUtils::getFilePathWithSuffixBufSize(dictDirPath,
+ DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
+ char tmpDirPath[tmpDirPathBufSize];
+ FileUtils::getFilePathWithSuffix(dictDirPath,
+ DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize,
+ tmpDirPath);
+ if (mkdir(tmpDirPath, S_IRWXU) == -1) {
+ AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
+ return false;
+ }
+ // Write trie file.
+ const BufferWithExtendableBuffer *buffers[] =
+ {&mExpandableHeaderBuffer, &mExpandableTrieBuffer};
+ if (!DictFileWritingUtils::flushBuffersToFileInDir(tmpDirPath,
+ Ver4DictConstants::TRIE_FILE_EXTENSION, buffers, 2 /* bufferCount */)) {
+ AKLOGE("Dictionary trie file %s/%s cannot be written.", tmpDirPath,
+ Ver4DictConstants::TRIE_FILE_EXTENSION);
+ return false;
+ }
+ // Write dictionary contents.
+ if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath)) {
+ AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
+ return false;
+ }
+ if (!mProbabilityDictContent.flushToFile(tmpDirPath)) {
+ AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
+ return false;
+ }
+ if (!mBigramDictContent.flushToFile(tmpDirPath)) {
+ AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
+ return false;
+ }
+ if (!mShortcutDictContent.flushToFile(tmpDirPath)) {
+ AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
+ return false;
+ }
+ // Remove existing dictionary.
+ if (!FileUtils::removeDirAndFiles(dictDirPath)) {
+ AKLOGE("Existing directory %s cannot be removed.", dictDirPath);
+ ASSERT(false);
+ return false;
+ }
+ // Rename temporary directory.
+ if (rename(tmpDirPath, dictDirPath) != 0) {
+ AKLOGE("%s cannot be renamed to %s", tmpDirPath, dictDirPath);
+ ASSERT(false);
+ return false;
+ }
+ return true;
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
index e468be5..0684bdd 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
@@ -89,10 +89,7 @@
return mIsUpdatable;
}
- bool flush(const char *const dictDirPath) {
- // TODO: Implement.
- return false;
- }
+ bool flush(const char *const dictDirPath) const;
private:
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
index 40f7d1f..2de8a60 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
@@ -17,12 +17,12 @@
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include <cstdio>
-#include <cstring>
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
namespace latinime {
@@ -36,9 +36,9 @@
return createEmptyV3DictFile(filePath, attributeMap);
case 4:
return createEmptyV4DictFile(filePath, attributeMap);
- return false;
default:
- // Only version 3 dictionary is supported for now.
+ AKLOGE("Cannot create dictionary %s because format version %d is not supported.",
+ filePath, dictVersion);
return false;
}
}
@@ -54,12 +54,13 @@
BufferWithExtendableBuffer bodyBuffer(
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE);
if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(&bodyBuffer, 0 /* rootPos */)) {
+ AKLOGE("Empty ver3 dictionary structure cannot be created on memory.");
return false;
}
return flushAllHeaderAndBodyToFile(filePath, &headerBuffer, &bodyBuffer);
}
-/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const filePath,
+/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers();
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
@@ -68,42 +69,59 @@
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(
dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) {
+ AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
return false;
}
- return dictBuffers.get()->flush(filePath);
+ return dictBuffers.get()->flush(dirPath);
}
/* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath,
BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) {
- const int tmpFileNameBufSize = strlen(filePath)
- + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */;
+ const int tmpFileNameBufSize = FileUtils::getFilePathWithSuffixBufSize(filePath,
+ TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
// Name of a temporary file used for writing that is a connected string of original name and
// TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE.
char tmpFileName[tmpFileNameBufSize];
- snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", filePath,
- TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
- FILE *const file = fopen(tmpFileName, "wb");
+ FileUtils::getFilePathWithSuffix(filePath, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE,
+ tmpFileNameBufSize, tmpFileName);
+ const BufferWithExtendableBuffer *buffers[] = {dictHeader, dictBody};
+ if (!DictFileWritingUtils::flushBuffersToFile(tmpFileName, buffers, 2 /* bufferCount */)) {
+ AKLOGE("Dictionary structure cannot be written to %s.", tmpFileName);
+ return false;
+ }
+ if (rename(tmpFileName, filePath) != 0) {
+ AKLOGE("Dictionary file %s cannot be renamed to %s", tmpFileName, filePath);;
+ }
+ return true;
+}
+
+/* static */ bool DictFileWritingUtils::flushBuffersToFileInDir(const char *const dirPath,
+ const char *const fileName, const BufferWithExtendableBuffer **const buffers,
+ const int bufferCount) {
+ const int filePathBufSize = FileUtils::getFilePathBufSize(dirPath, fileName);
+ char filePath[filePathBufSize];
+ FileUtils::getFilePath(dirPath, fileName, filePathBufSize, filePath);
+ return flushBuffersToFile(filePath, buffers, bufferCount);
+}
+
+/* static */ bool DictFileWritingUtils::flushBuffersToFile(const char *const filePath,
+ const BufferWithExtendableBuffer **const buffers, const int bufferCount) {
+ FILE *const file = fopen(filePath, "wb");
if (!file) {
- AKLOGE("Dictionary file %s cannot be opened.", tmpFileName);
+ AKLOGE("File %s cannot be opened.", filePath);
ASSERT(false);
return false;
}
- // Write the dictionary header.
- if (!writeBufferToFile(file, dictHeader)) {
- remove(tmpFileName);
- AKLOGE("Dictionary header cannot be written. size: %d", dictHeader->getTailPosition());
- ASSERT(false);
- return false;
- }
- // Write the dictionary body.
- if (!writeBufferToFile(file, dictBody)) {
- remove(tmpFileName);
- AKLOGE("Dictionary body cannot be written. size: %d", dictBody->getTailPosition());
- ASSERT(false);
- return false;
+ for (int i = 0; i < bufferCount; ++i) {
+ if (!writeBufferToFile(file, buffers[i])) {
+ remove(filePath);
+ AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath,
+ buffers[i]->getTailPosition());
+ ASSERT(false);
+ return false;
+ }
}
fclose(file);
- rename(tmpFileName, filePath);
return true;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
index 3291f98..980a1ff 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
@@ -28,6 +28,8 @@
class DictFileWritingUtils {
public:
+ static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
+
static bool createEmptyDictFile(const char *const filePath, const int dictVersion,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
@@ -35,17 +37,21 @@
BufferWithExtendableBuffer *const dictHeader,
BufferWithExtendableBuffer *const dictBody);
+ static bool flushBuffersToFileInDir(const char *const dirPath, const char *const fileName,
+ const BufferWithExtendableBuffer **const buffers, const int bufferCount);
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
- static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
-
static bool createEmptyV3DictFile(const char *const filePath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool createEmptyV4DictFile(const char *const filePath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
+ static bool flushBuffersToFile(const char *const filePath,
+ const BufferWithExtendableBuffer **const buffers, const int bufferCount);
+
static bool writeBufferToFile(FILE *const file,
const BufferWithExtendableBuffer *const buffer);
};
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
new file mode 100644
index 0000000..1748d5a
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/utils/file_utils.h"
+
+#include <cstdio>
+#include <cstring>
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+namespace latinime {
+
+// Returns -1 on error.
+/* static */ int FileUtils::getFileSize(const char *const filePath) {
+ const int fd = open(filePath, O_RDONLY);
+ if (fd == -1) {
+ return -1;
+ }
+ struct stat statBuf;
+ if (fstat(fd, &statBuf) != 0) {
+ close(fd);
+ return -1;
+ }
+ close(fd);
+ return static_cast<int>(statBuf.st_size);
+}
+
+// Remove a directory and all files in the directory.
+/* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) {
+ DIR *const dir = opendir(dirPath);
+ if (dir == NULL) {
+ AKLOGE("Cannot open dir %s.", dirPath);
+ return true;
+ }
+ struct dirent *dirent;
+ while ((dirent = readdir(dir)) != NULL) {
+ if (dirent->d_type != DT_REG) {
+ continue;
+ }
+ const int filePathBufSize = getFilePathBufSize(dirPath, dirent->d_name);
+ char filePath[filePathBufSize];
+ getFilePath(dirPath, dirent->d_name, filePathBufSize, filePath);
+ if (remove(filePath) != 0) {
+ AKLOGE("Cannot remove file %s.", filePath);
+ return false;
+ }
+ }
+ if (remove(dirPath) != 0) {
+ AKLOGE("Cannot remove directory %s.", dirPath);
+ return false;
+ }
+ return true;
+}
+
+/* static */ int FileUtils::getFilePathWithSuffixBufSize(const char *const filePath,
+ const char *const suffix) {
+ return strlen(filePath) + strlen(suffix) + 1 /* terminator */;
+}
+
+/* static */ void FileUtils::getFilePathWithSuffix(const char *const filePath,
+ const char *const suffix, const int filePathBufSize, char *const outFilePath) {
+ snprintf(outFilePath, filePathBufSize, "%s%s", filePath, suffix);
+}
+
+/* static */ int FileUtils::getFilePathBufSize(const char *const dirPath,
+ const char *const fileName) {
+ return strlen(dirPath) + 1 /* '/' */ + strlen(fileName) + 1 /* terminator */;
+}
+
+/* static */ void FileUtils::getFilePath(const char *const dirPath, const char *const fileName,
+ const int filePathBufSize, char *const outFilePath) {
+ snprintf(outFilePath, filePathBufSize, "%s/%s", dirPath, fileName);
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
index 59b894f..fc27aee 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
@@ -17,11 +17,6 @@
#ifndef LATINIME_FILE_UTILS_H
#define LATINIME_FILE_UTILS_H
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-
#include "defines.h"
namespace latinime {
@@ -29,19 +24,20 @@
class FileUtils {
public:
// Returns -1 on error.
- static int getFileSize(const char *const filePath) {
- const int fd = open(filePath, O_RDONLY);
- if (fd == -1) {
- return -1;
- }
- struct stat statBuf;
- if (fstat(fd, &statBuf) != 0) {
- close(fd);
- return -1;
- }
- close(fd);
- return static_cast<int>(statBuf.st_size);
- }
+ static int getFileSize(const char *const filePath);
+
+ // Remove a directory and all files in the directory.
+ static bool removeDirAndFiles(const char *const dirPath);
+
+ static int getFilePathWithSuffixBufSize(const char *const filePath, const char *const suffix);
+
+ static void getFilePathWithSuffix(const char *const filePath, const char *const suffix,
+ const int filePathBufSize, char *const outFilePath);
+
+ static int getFilePathBufSize(const char *const dirPath, const char *const fileName);
+
+ static void getFilePath(const char *const dirPath, const char *const fileName,
+ const int filePathBufSize, char *const outFilePath);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(FileUtils);
diff --git a/tests/src/com/android/inputmethod/latin/Ver4BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/Ver4BinaryDictionaryTests.java
index ad57e4c..15d990c 100644
--- a/tests/src/com/android/inputmethod/latin/Ver4BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/Ver4BinaryDictionaryTests.java
@@ -18,29 +18,21 @@
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
-import android.util.Log;
import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils;
-import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver4DictEncoder;
-import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Locale;
+import java.util.Map;
// TODO: Add a test to evaluate the speed of operations of Ver4 dictionary.
@LargeTest
public class Ver4BinaryDictionaryTests extends AndroidTestCase {
- private static final String TAG = Ver4BinaryDictionaryTests.class.getSimpleName();
private static final String TEST_LOCALE = "test";
- private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
- new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */);
+ private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
@Override
protected void setUp() throws Exception {
@@ -52,42 +44,35 @@
super.tearDown();
}
- // TODO: remove after native code support dictionary creation.
- private DictionaryOptions getDictionaryOptions(final String id, final String version) {
- final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(),
- false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */);
- options.mAttributes.put("version", version);
- options.mAttributes.put("dictionary", id);
- return options;
- }
-
- // TODO: remove after native code support dictionary creation.
- private File getTrieFile(final String id, final String version) {
- return new File(getContext().getCacheDir() + "/" + id + "." + version,
- TEST_LOCALE + "." + version + FormatSpec.TRIE_FILE_EXTENSION);
+ // Note that dictVersion is different from dictionary format version and it never affects the
+ // dictionary format.
+ // TODO: Rename dictVersion to understandable name such as dictRevision.
+ private File createEmptyDictionaryAndGetTrieFile(final String dictVersion) throws IOException {
+ final File file = File.createTempFile(dictVersion, TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
+ file.delete();
+ file.mkdir();
+ Map<String, String> attributeMap = new HashMap<String, String>();
+ attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+ 4 /* dictVersion */, attributeMap)) {
+ return new File(file, FormatSpec.TRIE_FILE_EXTENSION);
+ } else {
+ throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
+ + FormatSpec.TRIE_FILE_EXTENSION + " cannot be created.");
+ }
}
public void testIsValidDictionary() {
final String dictVersion = Long.toString(System.currentTimeMillis());
- final File trieFile = getTrieFile(TEST_LOCALE, dictVersion);
-
- BinaryDictionary binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
- 0 /* offset */, trieFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- assertFalse(binaryDictionary.isValidDictionary());
-
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- getDictionaryOptions(TEST_LOCALE, dictVersion));
- final DictEncoder encoder = new Ver4DictEncoder(getContext().getCacheDir());
+ File trieFile = null;
try {
- encoder.writeDictionary(dict, FORMAT_OPTIONS);
+ trieFile = createEmptyDictionaryAndGetTrieFile(dictVersion);
} catch (IOException e) {
- Log.e(TAG, "IOException while writing dictionary", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Unsupported format", e);
+ fail("IOException while writing an initial dictionary : " + e);
}
-
- binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
+ final BinaryDictionary binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
0 /* offset */, trieFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
assertTrue(binaryDictionary.isValidDictionary());
@@ -96,27 +81,21 @@
// TODO: Add large tests.
public void testReadProbability() {
final String dictVersion = Long.toString(System.currentTimeMillis());
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- getDictionaryOptions(TEST_LOCALE, dictVersion));
-
- final int frequency = 100;
- dict.add("a", frequency, null, false /* isNotAWord */);
- dict.add("aaa", frequency, null, false /* isNotAWord */);
- dict.add("ab", frequency, null, false /* isNotAWord */);
-
- final DictEncoder encoder = new Ver4DictEncoder(getContext().getCacheDir());
+ File trieFile = null;
try {
- encoder.writeDictionary(dict, FORMAT_OPTIONS);
+ trieFile = createEmptyDictionaryAndGetTrieFile(dictVersion);
} catch (IOException e) {
- Log.e(TAG, "IOException while writing dictionary", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Unsupported format", e);
+ fail("IOException while writing an initial dictionary : " + e);
}
- final File trieFile = getTrieFile(TEST_LOCALE, dictVersion);
final BinaryDictionary binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
0 /* offset */, trieFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- assertTrue(binaryDictionary.isValidDictionary());
+
+ final int frequency = 100;
+ binaryDictionary.addUnigramWord("a", frequency);
+ binaryDictionary.addUnigramWord("aaa", frequency);
+ binaryDictionary.addUnigramWord("ab", frequency);
+
assertEquals(frequency, binaryDictionary.getFrequency("a"));
assertEquals(frequency, binaryDictionary.getFrequency("aaa"));
assertEquals(frequency, binaryDictionary.getFrequency("ab"));
@@ -132,40 +111,32 @@
// TODO: Add large tests.
public void testReadBigrams() {
final String dictVersion = Long.toString(System.currentTimeMillis());
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- getDictionaryOptions(TEST_LOCALE, dictVersion));
-
- final int unigramFrequency = 1;
- final int bigramFrequency0 = 150;
- final int bigramFrequency1 = 1;
- final int bigramFrequency2 = 255;
- dict.add("a", unigramFrequency, null, false /* isNotAWord */);
- dict.add("aaa", unigramFrequency, null, false /* isNotAWord */);
- dict.add("ab", unigramFrequency, null, false /* isNotAWord */);
- dict.setBigram("a", "aaa", bigramFrequency0);
- dict.setBigram("a", "ab", bigramFrequency1);
- dict.setBigram("aaa", "ab", bigramFrequency2);
-
- final DictEncoder encoder = new Ver4DictEncoder(getContext().getCacheDir());
+ File trieFile = null;
try {
- encoder.writeDictionary(dict, FORMAT_OPTIONS);
+ trieFile = createEmptyDictionaryAndGetTrieFile(dictVersion);
} catch (IOException e) {
- Log.e(TAG, "IOException while writing dictionary", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Unsupported format", e);
+ fail("IOException while writing an initial dictionary : " + e);
}
- final File trieFile = getTrieFile(TEST_LOCALE, dictVersion);
final BinaryDictionary binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
0 /* offset */, trieFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- assertTrue(binaryDictionary.isValidDictionary());
+ final int unigramFrequency = 1;
+ final int bigramFrequency0 = 10;
+ final int bigramFrequency1 = 1;
+ final int bigramFrequency2 = 15;
+ binaryDictionary.addUnigramWord("a", unigramFrequency);
+ binaryDictionary.addUnigramWord("aaa", unigramFrequency);
+ binaryDictionary.addUnigramWord("ab", unigramFrequency);
+ binaryDictionary.addBigramWords("a", "aaa", bigramFrequency0);
+ binaryDictionary.addBigramWords("a", "ab", bigramFrequency1);
+ binaryDictionary.addBigramWords("aaa", "ab", bigramFrequency2);
- assertEquals(getCalculatedBigramProbabiliy(binaryDictionary, unigramFrequency,
+ assertEquals(binaryDictionary.calculateProbability(unigramFrequency,
bigramFrequency0), binaryDictionary.getBigramProbability("a", "aaa"));
- assertEquals(getCalculatedBigramProbabiliy(binaryDictionary, unigramFrequency,
+ assertEquals(binaryDictionary.calculateProbability(unigramFrequency,
bigramFrequency1), binaryDictionary.getBigramProbability("a", "ab"));
- assertEquals(getCalculatedBigramProbabiliy(binaryDictionary, unigramFrequency,
+ assertEquals(binaryDictionary.calculateProbability(unigramFrequency,
bigramFrequency2), binaryDictionary.getBigramProbability("aaa", "ab"));
assertFalse(binaryDictionary.isValidBigram("aaa", "a"));
@@ -176,21 +147,15 @@
// TODO: Add large tests.
public void testWriteUnigrams() {
final String dictVersion = Long.toString(System.currentTimeMillis());
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- getDictionaryOptions(TEST_LOCALE, dictVersion));
- final DictEncoder encoder = new Ver4DictEncoder(getContext().getCacheDir());
+ File trieFile = null;
try {
- encoder.writeDictionary(dict, FORMAT_OPTIONS);
+ trieFile = createEmptyDictionaryAndGetTrieFile(dictVersion);
} catch (IOException e) {
- Log.e(TAG, "IOException while writing dictionary", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Unsupported format", e);
+ fail("IOException while writing an initial dictionary : " + e);
}
- final File trieFile = getTrieFile(TEST_LOCALE, dictVersion);
final BinaryDictionary binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
0 /* offset */, trieFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- assertTrue(binaryDictionary.isValidDictionary());
final int probability = 100;
binaryDictionary.addUnigramWord("aaa", probability);
@@ -208,25 +173,18 @@
public void testWriteBigrams() {
final String dictVersion = Long.toString(System.currentTimeMillis());
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- getDictionaryOptions(TEST_LOCALE, dictVersion));
- final DictEncoder encoder = new Ver4DictEncoder(getContext().getCacheDir());
+ File trieFile = null;
try {
- encoder.writeDictionary(dict, FORMAT_OPTIONS);
+ trieFile = createEmptyDictionaryAndGetTrieFile(dictVersion);
} catch (IOException e) {
- Log.e(TAG, "IOException while writing dictionary", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Unsupported format", e);
+ fail("IOException while writing an initial dictionary : " + e);
}
- final File trieFile = getTrieFile(TEST_LOCALE, dictVersion);
final BinaryDictionary binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
0 /* offset */, trieFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- assertTrue(binaryDictionary.isValidDictionary());
final int unigramProbability = 100;
final int bigramProbability = 10;
- final int updatedBigramProbability = 15;
binaryDictionary.addUnigramWord("aaa", unigramProbability);
binaryDictionary.addUnigramWord("abb", unigramProbability);
binaryDictionary.addUnigramWord("bcc", unigramProbability);
@@ -249,21 +207,15 @@
public void testRemoveBigramWords() {
final String dictVersion = Long.toString(System.currentTimeMillis());
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- getDictionaryOptions(TEST_LOCALE, dictVersion));
- final DictEncoder encoder = new Ver4DictEncoder(getContext().getCacheDir());
+ File trieFile = null;
try {
- encoder.writeDictionary(dict, FORMAT_OPTIONS);
+ trieFile = createEmptyDictionaryAndGetTrieFile(dictVersion);
} catch (IOException e) {
- Log.e(TAG, "IOException while writing dictionary", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Unsupported format", e);
+ fail("IOException while writing an initial dictionary : " + e);
}
- final File trieFile = getTrieFile(TEST_LOCALE, dictVersion);
final BinaryDictionary binaryDictionary = new BinaryDictionary(trieFile.getAbsolutePath(),
0 /* offset */, trieFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- assertTrue(binaryDictionary.isValidDictionary());
final int unigramProbability = 100;
final int bigramProbability = 10;