Pseudolocale support for grammatical gender
Default behavior is generate grammatical gender strings with different
prefix when pseudolocale is enabled. Also extend two specifiers to
support generate grammatical gender string for a specific gender and for
a spefific ratio. go/pseudolocale-support-for-grammatical-gender
Bug: b/272626712
Test: Added and verified affected atests pass
Change-Id: I6b7514425898facb68f0b1f6fb09e4f87978c03d
diff --git a/tools/aapt2/compile/PseudolocaleGenerator.cpp b/tools/aapt2/compile/PseudolocaleGenerator.cpp
index 09a8560..8143052 100644
--- a/tools/aapt2/compile/PseudolocaleGenerator.cpp
+++ b/tools/aapt2/compile/PseudolocaleGenerator.cpp
@@ -16,11 +16,15 @@
#include "compile/PseudolocaleGenerator.h"
+#include <stdint.h>
+
#include <algorithm>
+#include <random>
#include "ResourceTable.h"
#include "ResourceValues.h"
#include "ValueVisitor.h"
+#include "androidfw/ResourceTypes.h"
#include "androidfw/Util.h"
#include "compile/Pseudolocalizer.h"
#include "util/Util.h"
@@ -293,8 +297,85 @@
Pseudolocalizer localizer_;
};
+class GrammaticalGenderVisitor : public ValueVisitor {
+ public:
+ std::unique_ptr<Value> value;
+ std::unique_ptr<Item> item;
+
+ GrammaticalGenderVisitor(android::StringPool* pool, uint8_t grammaticalInflection)
+ : pool_(pool), grammaticalInflection_(grammaticalInflection) {
+ }
+
+ void Visit(Plural* plural) override {
+ CloningValueTransformer cloner(pool_);
+ std::unique_ptr<Plural> grammatical_gendered = util::make_unique<Plural>();
+ for (size_t i = 0; i < plural->values.size(); i++) {
+ if (plural->values[i]) {
+ GrammaticalGenderVisitor sub_visitor(pool_, grammaticalInflection_);
+ plural->values[i]->Accept(&sub_visitor);
+ if (sub_visitor.item) {
+ grammatical_gendered->values[i] = std::move(sub_visitor.item);
+ } else {
+ grammatical_gendered->values[i] = plural->values[i]->Transform(cloner);
+ }
+ }
+ }
+ grammatical_gendered->SetSource(plural->GetSource());
+ grammatical_gendered->SetWeak(true);
+ value = std::move(grammatical_gendered);
+ }
+
+ std::string AddGrammaticalGenderPrefix(const std::string_view& original_string) {
+ std::string result;
+ switch (grammaticalInflection_) {
+ case android::ResTable_config::GRAMMATICAL_GENDER_MASCULINE:
+ result = std::string("(M)") + std::string(original_string);
+ break;
+ case android::ResTable_config::GRAMMATICAL_GENDER_FEMININE:
+ result = std::string("(F)") + std::string(original_string);
+ break;
+ case android::ResTable_config::GRAMMATICAL_GENDER_NEUTER:
+ result = std::string("(N)") + std::string(original_string);
+ break;
+ default:
+ result = std::string(original_string);
+ break;
+ }
+ return result;
+ }
+
+ void Visit(String* string) override {
+ std::string prefixed_string = AddGrammaticalGenderPrefix(std::string(*string->value));
+ std::unique_ptr<String> grammatical_gendered =
+ util::make_unique<String>(pool_->MakeRef(prefixed_string));
+ grammatical_gendered->SetSource(string->GetSource());
+ grammatical_gendered->SetWeak(true);
+ item = std::move(grammatical_gendered);
+ }
+
+ void Visit(StyledString* string) override {
+ std::string prefixed_string = AddGrammaticalGenderPrefix(std::string(string->value->value));
+ android::StyleString new_string;
+ new_string.str = std::move(prefixed_string);
+ for (const android::StringPool::Span& span : string->value->spans) {
+ new_string.spans.emplace_back(android::Span{*span.name, span.first_char, span.last_char});
+ }
+ std::unique_ptr<StyledString> grammatical_gendered =
+ util::make_unique<StyledString>(pool_->MakeRef(new_string));
+ grammatical_gendered->SetSource(string->GetSource());
+ grammatical_gendered->SetWeak(true);
+ item = std::move(grammatical_gendered);
+ }
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(GrammaticalGenderVisitor);
+ android::StringPool* pool_;
+ uint8_t grammaticalInflection_;
+};
+
ConfigDescription ModifyConfigForPseudoLocale(const ConfigDescription& base,
- Pseudolocalizer::Method m) {
+ Pseudolocalizer::Method m,
+ uint8_t grammaticalInflection) {
ConfigDescription modified = base;
switch (m) {
case Pseudolocalizer::Method::kAccent:
@@ -313,12 +394,64 @@
default:
break;
}
+ modified.grammaticalInflection = grammaticalInflection;
return modified;
}
+void GrammaticalGender(ResourceConfigValue* original_value,
+ ResourceConfigValue* localized_config_value, android::StringPool* pool,
+ ResourceEntry* entry, const Pseudolocalizer::Method method,
+ uint8_t grammaticalInflection) {
+ GrammaticalGenderVisitor visitor(pool, grammaticalInflection);
+ localized_config_value->value->Accept(&visitor);
+
+ std::unique_ptr<Value> grammatical_gendered_value;
+ if (visitor.value) {
+ grammatical_gendered_value = std::move(visitor.value);
+ } else if (visitor.item) {
+ grammatical_gendered_value = std::move(visitor.item);
+ }
+ if (!grammatical_gendered_value) {
+ return;
+ }
+
+ ConfigDescription config =
+ ModifyConfigForPseudoLocale(original_value->config, method, grammaticalInflection);
+
+ ResourceConfigValue* grammatical_gendered_config_value =
+ entry->FindOrCreateValue(config, original_value->product);
+ if (!grammatical_gendered_config_value->value) {
+ // Only use auto-generated pseudo-localization if none is defined.
+ grammatical_gendered_config_value->value = std::move(grammatical_gendered_value);
+ }
+}
+
+const uint32_t MASK_MASCULINE = 1; // Bit mask for masculine
+const uint32_t MASK_FEMININE = 2; // Bit mask for feminine
+const uint32_t MASK_NEUTER = 4; // Bit mask for neuter
+
+void GrammaticalGenderIfNeeded(ResourceConfigValue* original_value, ResourceConfigValue* new_value,
+ android::StringPool* pool, ResourceEntry* entry,
+ const Pseudolocalizer::Method method, uint32_t gender_state) {
+ if (gender_state & MASK_FEMININE) {
+ GrammaticalGender(original_value, new_value, pool, entry, method,
+ android::ResTable_config::GRAMMATICAL_GENDER_FEMININE);
+ }
+
+ if (gender_state & MASK_MASCULINE) {
+ GrammaticalGender(original_value, new_value, pool, entry, method,
+ android::ResTable_config::GRAMMATICAL_GENDER_MASCULINE);
+ }
+
+ if (gender_state & MASK_NEUTER) {
+ GrammaticalGender(original_value, new_value, pool, entry, method,
+ android::ResTable_config::GRAMMATICAL_GENDER_NEUTER);
+ }
+}
+
void PseudolocalizeIfNeeded(const Pseudolocalizer::Method method,
ResourceConfigValue* original_value, android::StringPool* pool,
- ResourceEntry* entry) {
+ ResourceEntry* entry, uint32_t gender_state, bool gender_flag) {
Visitor visitor(pool, method);
original_value->value->Accept(&visitor);
@@ -333,8 +466,8 @@
return;
}
- ConfigDescription config_with_accent =
- ModifyConfigForPseudoLocale(original_value->config, method);
+ ConfigDescription config_with_accent = ModifyConfigForPseudoLocale(
+ original_value->config, method, android::ResTable_config::GRAMMATICAL_GENDER_ANY);
ResourceConfigValue* new_config_value =
entry->FindOrCreateValue(config_with_accent, original_value->product);
@@ -342,6 +475,9 @@
// Only use auto-generated pseudo-localization if none is defined.
new_config_value->value = std::move(localized_value);
}
+ if (gender_flag) {
+ GrammaticalGenderIfNeeded(original_value, new_config_value, pool, entry, method, gender_state);
+ }
}
// A value is pseudolocalizable if it does not define a locale (or is the default locale) and is
@@ -356,16 +492,71 @@
} // namespace
+bool ParseGenderValuesAndSaveState(const std::string& grammatical_gender_values,
+ uint32_t* gender_state, android::IDiagnostics* diag) {
+ std::vector<std::string> values = util::SplitAndLowercase(grammatical_gender_values, ',');
+ for (size_t i = 0; i < values.size(); i++) {
+ if (values[i].length() != 0) {
+ if (values[i] == "f") {
+ *gender_state |= MASK_FEMININE;
+ } else if (values[i] == "m") {
+ *gender_state |= MASK_MASCULINE;
+ } else if (values[i] == "n") {
+ *gender_state |= MASK_NEUTER;
+ } else {
+ diag->Error(android::DiagMessage() << "Invalid grammatical gender value: " << values[i]);
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+bool ParseGenderRatio(const std::string& grammatical_gender_ratio, float* gender_ratio,
+ android::IDiagnostics* diag) {
+ const char* input = grammatical_gender_ratio.c_str();
+ char* endPtr;
+ errno = 0;
+ *gender_ratio = strtof(input, &endPtr);
+ if (endPtr == input || *endPtr != '\0' || errno == ERANGE || *gender_ratio < 0 ||
+ *gender_ratio > 1) {
+ diag->Error(android::DiagMessage()
+ << "Invalid grammatical gender ratio: " << grammatical_gender_ratio
+ << ", must be a real number between 0 and 1");
+ return false;
+ }
+ return true;
+}
+
bool PseudolocaleGenerator::Consume(IAaptContext* context, ResourceTable* table) {
+ uint32_t gender_state = 0;
+ if (!ParseGenderValuesAndSaveState(grammatical_gender_values_, &gender_state,
+ context->GetDiagnostics())) {
+ return false;
+ }
+
+ float gender_ratio = 0;
+ if (!ParseGenderRatio(grammatical_gender_ratio_, &gender_ratio, context->GetDiagnostics())) {
+ return false;
+ }
+
+ std::random_device rd;
+ std::mt19937 gen(rd());
+ std::uniform_real_distribution<> distrib(0.0, 1.0);
+
for (auto& package : table->packages) {
for (auto& type : package->types) {
for (auto& entry : type->entries) {
+ bool gender_flag = false;
+ if (distrib(gen) < gender_ratio) {
+ gender_flag = true;
+ }
std::vector<ResourceConfigValue*> values = entry->FindValuesIf(IsPseudolocalizable);
for (ResourceConfigValue* value : values) {
PseudolocalizeIfNeeded(Pseudolocalizer::Method::kAccent, value, &table->string_pool,
- entry.get());
+ entry.get(), gender_state, gender_flag);
PseudolocalizeIfNeeded(Pseudolocalizer::Method::kBidi, value, &table->string_pool,
- entry.get());
+ entry.get(), gender_state, gender_flag);
}
}
}