Be ready for changes in bf-translations repository
authorSergey Sharybin <sergey.vfx@gmail.com>
Fri, 8 Nov 2013 20:44:48 +0000 (20:44 +0000)
committerSergey Sharybin <sergey.vfx@gmail.com>
Fri, 8 Nov 2013 20:44:48 +0000 (20:44 +0000)
Made it so if there's release/datafiles/locale/po
folder, then all the .po files will be converted
to .mo at blender compile time and installed to
an appropriate location.

Uses small own implementation msgfmt which is
based on msgfmt.py from Python project, but also
supports contexts.

There's no functional changes for until we've
switched to use source .po files instead of
pre-compiled .mo.

P.S. Well, there's one change which is msgfmt.cc
     being compiled even if it's not used, but
     would rather not clutter code with checks
     since pretty soon we'll use this program
     anyway.

SConstruct
build_files/cmake/macros.cmake
intern/locale/CMakeLists.txt
intern/locale/SConscript
intern/locale/msgfmt.cc [new file with mode: 0644]
source/creator/CMakeLists.txt

index 4a08b691e077d80469d670038d1195be5f8bf2db..29d4d65b167b187eb96fa32b2d9789fba2181b0a 100644 (file)
@@ -944,6 +944,9 @@ if env['OURPLATFORM']!='darwin':
         def check_path(path, member):
             return (member in path.split(os.sep))
 
+        po_dir = os.path.join("release", "datafiles", "locale", "po")
+        need_compile_mo = os.path.exists(po_dir)
+
         for intpath in internationalpaths:
             for dp, dn, df in os.walk(intpath):
                 if '.svn' in dn:
@@ -952,7 +955,10 @@ if env['OURPLATFORM']!='darwin':
                     dn.remove('_svn')
 
                 # we only care about release/datafiles/fonts, release/datafiles/locales
-                if check_path(dp, "fonts") or check_path(dp, "locale"):
+                if check_path(dp, "locale"):
+                    if need_compile_mo and check_path(dp, "po"):
+                        continue
+                elif check_path(dp, "fonts"):
                     pass
                 else:
                     continue
@@ -966,6 +972,19 @@ if env['OURPLATFORM']!='darwin':
                     env.Execute(Mkdir(dir))
                 scriptinstall.append(env.Install(dir=dir,source=source))
 
+        if need_compile_mo:
+            for f in os.listdir(po_dir):
+                if not f.endswith(".po"):
+                    continue
+
+                locale_name = os.path.splitext(f)[0]
+
+                mo_file = os.path.join(B.root_build_dir, "locale", locale_name + ".mo")
+
+                dir = os.path.join(env['BF_INSTALLDIR'], VERSION)
+                dir = os.path.join(dir, "datafiles", "locale", locale_name, "LC_MESSAGES")
+                scriptinstall.append(env.InstallAs(os.path.join(dir, "blender.mo"), mo_file))
+
 #-- icons
 if env['OURPLATFORM']=='linux':
     iconlist = []
index fdc0fb63c8eb57114af288303870001412c806ce..7a06352ac70282c945bd4c85b1c1c9104d795dcb 100644 (file)
@@ -870,3 +870,33 @@ macro(svg_to_png
        unset(_file_to)
 
 endmacro()
+
+macro(msgfmt_simple
+      file_from
+      list_to_add)
+
+       # remove ../'s
+       get_filename_component(_file_from_we ${file_from} NAME_WE)
+
+       get_filename_component(_file_from ${file_from} REALPATH)
+       get_filename_component(_file_to ${CMAKE_CURRENT_BINARY_DIR}/${_file_from_we}.mo REALPATH)
+
+       list(APPEND ${list_to_add} ${_file_to})
+
+       get_filename_component(_file_to_path ${_file_to} PATH)
+
+       add_custom_command(
+               OUTPUT  ${_file_to}
+               COMMAND ${CMAKE_COMMAND} -E make_directory ${_file_to_path}
+               COMMAND ${CMAKE_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/msgfmt ${_file_from} ${_file_to}
+               DEPENDS msgfmt)
+
+       message("${CMAKE_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/msgfmt ${_file_from} ${_file_to}")
+
+       set_source_files_properties(${_file_to} PROPERTIES GENERATED TRUE)
+
+       unset(_file_from_we)
+       unset(_file_from)
+       unset(_file_to)
+       unset(_file_to_path)
+endmacro()
index 1f14a0e7a6a11c6bcd7a5052e21ae8afd6c6c4e0..3599aa6854525281cf9bb5a3e065b63988eb0043 100644 (file)
@@ -45,3 +45,11 @@ if(WITH_INTERNATIONAL)
 endif()
 
 blender_add_lib(bf_intern_locale "${SRC}" "${INC}" "${INC_SYS}")
+
+# -----------------------------------------------------------------------------
+# Build msgfmt executable
+set(MSFFMT_SRC
+       msgfmt.cc
+)
+
+add_executable(msgfmt ${MSFFMT_SRC})
index f60bd90fb387e98648292063d7e785abb0a0e12c..546fd3e8b401f6436bdc6a39655b0b2726fbaee5 100644 (file)
@@ -37,3 +37,49 @@ if env['WITH_BF_INTERNATIONAL']:
     incs += ' ' + env['BF_BOOST_INC']
 
 env.BlenderLib( 'bf_intern_locale', sources, Split(incs), defs, libtype=['intern','player'], priority=[10, 185])
+
+if env['WITH_BF_INTERNATIONAL']:
+    import os
+    from os.path import dirname
+
+    def normpath(path):
+        return os.path.abspath(os.path.normpath(path))
+
+    # build directory
+    source_dir = Dir('.').srcnode().path
+    root_build_dir = normpath(env['BF_BUILDDIR'])
+    root_source_dir = dirname(dirname(normpath(source_dir)))
+    po_dir = os.path.join(root_source_dir, "release", "datafiles", "locale", "po")
+    build_dir = os.path.join(root_build_dir, 'locale')
+
+    if os.path.exists(po_dir):
+        # create directory if needed
+        if not os.path.exists(build_dir):
+            os.makedirs(build_dir)
+
+        msgfmt_tool = env.Clone()
+        targetpath = root_build_dir + '/msgfmt'
+        msgfmt_target = msgfmt_tool.Program(target = targetpath, source = ['msgfmt.cc'])
+
+        locale = env.Clone()
+
+        # dependencies
+        dependencies = [targetpath]
+
+        # add command for each locale
+        all_mo_files = []
+        for f in os.listdir(po_dir):
+            if not f.endswith(".po"):
+                continue
+
+            po_file = os.path.join(po_dir, f)
+            mo_file = os.path.join(build_dir, os.path.splitext(f)[0] + ".mo")
+
+            command = "\"%s\" \"%s\" \"%s\"" % (targetpath, po_file, mo_file)
+
+            locale.Command(mo_file, po_file, command)
+            locale.Depends(mo_file, dependencies)
+
+            all_mo_files.append(mo_file)
+
+        env.Depends("boost_locale_wrapper.cpp", all_mo_files)
diff --git a/intern/locale/msgfmt.cc b/intern/locale/msgfmt.cc
new file mode 100644 (file)
index 0000000..cd858cd
--- /dev/null
@@ -0,0 +1,366 @@
+// Written by Sergey Sharybin <sergey.vfx@gmail.com>
+// Added support for contexts
+//
+// Based on Python script msgfmt.py from Python source
+// code tree, which was written by Written by
+// Martin v. Löwis <loewis@informatik.hu-berlin.de>
+//
+// Generate binary message catalog from textual translation description.
+//
+// This program converts a textual Uniforum-style message catalog (.po file) into
+// a binary GNU catalog (.mo file).  This is essentially the same function as the
+// GNU msgfmt program, however, it is a simpler implementation.
+//
+// Usage: msgfmt input.po output.po
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <stdlib.h>
+#include <string>
+#include <vector>
+
+namespace {
+
+std::map<std::string, std::string> MESSAGES;
+
+bool starts_with(const std::string &string,
+                 const std::string &prefix) {
+  return prefix.size() <= string.size() &&
+         string.compare(0, prefix.size(), prefix) == 0;
+}
+
+std::string ltrim(const std::string &s) {
+  std::string result = s;
+  result.erase(result.begin(),
+               std::find_if(result.begin(),
+                            result.end(),
+                            std::not1(std::ptr_fun<int, int>(std::isspace))));
+  return result;
+}
+
+std::string rtrim(const std::string &s) {
+  std::string result = s;
+  result.erase(
+    std::find_if(result.rbegin(),
+                 result.rend(),
+                 std::not1(std::ptr_fun<int, int>(std::isspace))).base(),
+    result.end());
+  return result;
+}
+
+std::string trim(const std::string &s) {
+  return ltrim(rtrim(s));
+}
+
+std::string unescape(const std::string &s) {
+  std::string result;
+  std::string::const_iterator it = s.begin();
+  while (it != s.end()) {
+    char current_char = *it++;
+    if (current_char == '\\' && it != s.end()) {
+      char next_char = *it++;
+      if (next_char == '\\') {
+        current_char = '\\';
+      } else if (next_char == 'n') {
+        current_char = '\n';
+      } else if (next_char == 't') {
+        current_char = '\t';
+      } else {
+        current_char = next_char;
+      }
+    }
+    result += current_char;
+  }
+
+  if (result[0] == '"' && result[result.size() - 1] == '"') {
+    result = result.substr(1, result.size() - 2);
+  }
+
+  return result;
+}
+
+// Add a non-fuzzy translation to the dictionary.
+void add(const std::string &msgctxt,
+         const std::string &msgid,
+         const std::string &msgstr,
+         bool fuzzy) {
+  if (fuzzy == false && msgstr.empty() == false) {
+    if (msgctxt.empty()) {
+      MESSAGES[msgid] = msgstr;
+    } else {
+      MESSAGES[msgctxt + (char)0x04 + msgid] = msgstr;
+    }
+  }
+}
+
+template<typename TKey, typename TValue>
+void get_keys(std::map<TKey, TValue> map,
+              std::vector<TKey> *keys) {
+  for (typename std::map<TKey, TValue>::iterator it = map.begin();
+      it != map.end();
+      it++) {
+    keys->push_back(it->first);
+  }
+}
+
+std::string intToBytes(int value) {
+  std::string result;
+  for (unsigned int i = 0; i < sizeof(value); i++) {
+    result += (unsigned char) ((value >> (i * 8)) & 0xff);
+  }
+  return result;
+}
+
+typedef enum {
+  SECTION_NONE = 0,
+  SECTION_CTX  = 1,
+  SECTION_ID   = 2,
+  SECTION_STR  = 3
+} eSectionType;
+
+struct Offset {
+  unsigned int o1, l1, o2, l2;
+};
+
+// Return the generated output.
+std::string generate(void) {
+  // The keys are sorted in the .mo file
+  std::vector<std::string> keys;
+
+  // Get list of sorted keys.
+  get_keys(MESSAGES, &keys);
+  std::sort(keys.begin(), keys.end());
+
+  std::vector<Offset> offsets;
+  std::string ids = "", strs = "";
+  for (std::vector<std::string>::iterator it = keys.begin();
+       it != keys.end();
+       it++) {
+    std::string &id = *it;
+    // For each string, we need size and file offset.  Each string is NUL
+    // terminated; the NUL does not count into the size.
+    Offset offset = {(unsigned int) ids.size(),
+                     (unsigned int) id.size(),
+                     (unsigned int) strs.size(),
+                     (unsigned int) MESSAGES[id].size()};
+    offsets.push_back(offset);
+    ids += id + '\0';
+    strs += MESSAGES[id] + '\0';
+  }
+
+  // The header is 7 32-bit unsigned integers.  We don't use hash tables, so
+  // the keys start right after the index tables.
+  // translated string.
+  int keystart = 7 * 4 + 16 * keys.size();
+  // and the values start after the keys
+  int valuestart = keystart + ids.size();
+  std::vector<int> koffsets;
+  std::vector<int> voffsets;
+  // The string table first has the list of keys, then the list of values.
+  // Each entry has first the size of the string, then the file offset.
+  for (std::vector<Offset>::iterator it = offsets.begin();
+       it != offsets.end();
+       it++) {
+    Offset &offset = *it;
+    koffsets.push_back(offset.l1);
+    koffsets.push_back(offset.o1 + keystart);
+    voffsets.push_back(offset.l2);
+    voffsets.push_back(offset.o2 + valuestart);
+  }
+
+  std::vector<int> all_offsets;
+  all_offsets.reserve(koffsets.size() + voffsets.size());
+  all_offsets.insert(all_offsets.end(), koffsets.begin(), koffsets.end());
+  all_offsets.insert(all_offsets.end(), voffsets.begin(), voffsets.end());
+
+  std::string output = "";
+  output += intToBytes(0x950412de);  // Magic
+  output += intToBytes(0x0);  // Version
+  output += intToBytes(keys.size());  // # of entries
+  output += intToBytes(7 * 4);  // start of key index
+  output += intToBytes(7 * 4 + keys.size() * 8);  // start of value index
+  output += intToBytes(0);  // Size of hash table
+  output += intToBytes(0);  // Offset of hash table
+
+  for (std::vector<int>::iterator it = all_offsets.begin();
+       it != all_offsets.end();
+       it++) {
+    int offset = *it;
+    output += intToBytes(offset);
+  }
+
+  output += ids;
+  output += strs;
+
+  return output;
+}
+
+void make(const char *input_file_name,
+          const char *output_file_name) {
+  std::map<std::string, std::string> messages;
+
+  // Start off assuming Latin-1, so everything decodes without failure,
+  // until we know the exact encoding.
+  // TODO(sergey): Support encoding.
+  // const char *encoding = "latin-1";
+
+  eSectionType section = SECTION_NONE;
+  bool fuzzy = false;
+  bool is_plural = false;
+  std::string msgctxt, msgid, msgstr;
+
+  std::ifstream input_file_stream(input_file_name);
+
+  // Parse the catalog.
+  int lno = 0;
+  for (std::string l; getline(input_file_stream, l); ) {
+    lno++;
+    // If we get a comment line after a msgstr, this is a new entry.
+    if (l[0] == '#' && section == SECTION_STR) {
+      add(msgctxt, msgid, msgstr, fuzzy);
+      section = SECTION_NONE;
+      fuzzy = false;
+    }
+    // Record a fuzzy mark.
+    if (starts_with(l, "#,") && l.find("fuzzy") != std::string::npos) {
+      fuzzy = 1;
+    }
+    // Skip comments
+    if (l[0] == '#') {
+      continue;
+    }
+    // Now we are in a msgid section, output previous section.
+    if (starts_with(l, "msgctxt")) {
+      if (section == SECTION_STR) {
+        add(msgctxt, msgid, msgstr, fuzzy);
+      }
+      section = SECTION_CTX;
+      l = l.substr(7, l.size() - 7);
+      msgctxt = msgid = msgstr = "";
+    }
+    else if (starts_with(l, "msgid") && !starts_with(l, "msgid_plural")) {
+      if (section == SECTION_STR) {
+        add(msgctxt, msgid, msgstr, fuzzy);
+        msgctxt = "";
+        if (msgid == "") {
+#if 0
+          // See whether there is an encoding declaration.
+          p = HeaderParser();
+          charset = p.parsestr(msgstr.decode(encoding)).get_content_charset();
+          if (charset) {
+            encoding = charset;
+          }
+#else
+          // Not ported to C++ yet.
+          std::cerr << "Encoding declarations are not supported yet.\n"
+                    << std::endl;
+          abort();
+#endif
+        }
+      }
+      section = SECTION_ID;
+      l = l.substr(5, l.size() - 5);
+      msgid = msgstr = "";
+      is_plural = false;
+    } else if (starts_with(l, "msgid_plural")) {
+      // This is a message with plural forms.
+      if (section != SECTION_ID) {
+        std::cerr << "msgid_plural not preceeded by msgid on"
+                  << input_file_name << ":"
+                  << lno
+                  << std::endl;
+        abort();
+      }
+      l = l.substr(12, l.size() - 12);
+      msgid += '\0';  // separator of singular and plural
+      is_plural = true;
+    } else if (starts_with(l, "msgstr")) {
+      // Now we are in a msgstr section
+      section = SECTION_STR;
+      if (starts_with(l, "msgstr[")) {
+        if (is_plural == false) {
+          std::cerr << "plural without msgid_plural on "
+                    << input_file_name << ":"
+                    << lno
+                    << std::endl;
+          abort();
+        }
+        int bracket_position = l.find(']');
+        if (bracket_position == std::string::npos) {
+          std::cerr << "Syntax error on "
+                    << input_file_name << ":"
+                    << lno
+                    << std::endl;
+          abort();
+        }
+        l = l.substr(bracket_position, l.size() - bracket_position);
+        if (msgstr != "") {
+          msgstr += '\0';  // Separator of the various plural forms;
+        }
+      } else {
+        if (is_plural) {
+          std::cerr << "indexed msgstr required for plural on "
+                    << input_file_name << ":"
+                    << lno
+                    << std::endl;
+          abort();
+        }
+        l = l.substr(6, l.size() - 6);
+      }
+    }
+    // Skip empty lines.
+    l = trim(l);
+    if (l.empty()) {
+      continue;
+    }
+    l = unescape(l);
+    if (section == SECTION_CTX) {
+      // TODO(sergey): Support encoding.
+      // msgid += l.encode(encoding);
+      msgctxt += l;
+    }
+    else if (section == SECTION_ID) {
+      // TODO(sergey): Support encoding.
+      // msgid += l.encode(encoding);
+      msgid += l;
+    } else if (section == SECTION_STR) {
+      // TODO(sergey): Support encoding.
+      // msgstr += l.encode(encoding)
+      msgstr += l;
+    } else {
+      std::cerr << "Syntax error on "
+                << input_file_name << ":"
+                << lno
+                << std::endl;
+      abort();
+    }
+    // Add last entry
+    if (section == SECTION_STR) {
+      add(msgctxt, msgid, msgstr, fuzzy);
+    }
+  }
+
+  // Compute output
+  std::string output = generate();
+
+  std::ofstream output_file_stream(output_file_name,
+    std::ios::out | std::ios::binary);
+  output_file_stream << output;
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  if (argc != 3) {
+    printf("Usage: %s <input.po> <output.mo>\n", argv[0]);
+    return EXIT_FAILURE;
+  }
+  const char *input_file = argv[1];
+  const char *output_file = argv[2];
+
+  make(input_file, output_file);
+
+  return EXIT_SUCCESS;
+}
index 33d5c7dc90b2e48cc88e2ecf723223bf90edf5e1..b386312bb55fabbd5ab160adf547b729bea531bc 100644 (file)
@@ -318,11 +318,59 @@ endif()
 if(WITH_INTERNATIONAL)
        install(
                DIRECTORY
-                       ${CMAKE_SOURCE_DIR}/release/datafiles/locale
                        ${CMAKE_SOURCE_DIR}/release/datafiles/fonts
                DESTINATION ${TARGETDIR_VER}/datafiles
                PATTERN ".svn" EXCLUDE
        )
+
+       set(_locale_dir "${CMAKE_SOURCE_DIR}/release/datafiles/locale")
+
+       if(EXISTS "${_locale_dir}/po")
+               set(_locale_target_dir ${TARGETDIR_VER}/datafiles/locale)
+
+               file(GLOB _po_files "${_locale_dir}/po/*.po")
+               foreach(_po_file ${_po_files})
+                       msgfmt_simple(${_po_file} _all_mo_files)
+               endforeach()
+
+               # Create a custom target which will compile all po to mo
+               add_custom_target(
+                       locales
+                       DEPENDS ${_all_mo_files})
+
+               add_dependencies(blender locales)
+
+               # Generate INSTALL rules
+               install(
+                       FILES ${_locale_dir}/languages
+                       DESTINATION ${_locale_target_dir}
+               )
+
+               foreach(_mo_file ${_all_mo_files})
+                       get_filename_component(_locale_name ${_mo_file} NAME_WE)
+                       install(
+                               FILES ${_mo_file}
+                               DESTINATION ${_locale_target_dir}/${_locale_name}/LC_MESSAGES
+                               RENAME blender.mo
+                       )
+                       unset(_locale_name)
+               endforeach()
+
+               unset(_all_mo_files)
+               unset(_po_files)
+               unset(_po_file)
+               unset(_mo_file)
+               unset(_locale_target_dir)
+       else()
+               install(
+                       DIRECTORY
+                               ${_locale_dir}
+                       DESTINATION ${TARGETDIR_VER}/datafiles
+                       PATTERN ".svn" EXCLUDE
+               )
+       endif()
+
+       unset(_locale_dir)
 endif()
 
 # color management