ICU-20524 Adding variables for relative file replacement directory.

Also fixes some bugs with file replacement.

diff --git a/.ci-builds/data-filter.json b/.ci-builds/data-filter.json
index 72d5aa4abd9e0933ff3498e34b07ea365be0dd9d..177478acff228d716678b8095bb04510b572c9c1 100644 (file)
--- a/.ci-builds/data-filter.json
+++ b/.ci-builds/data-filter.json
@@ -21,5 +21,16 @@
       },
       "rules": ["+/*"]
     }
-  ]
+  ],
+// Test fileReplacements (relative path)
+  "fileReplacements": {
+    "directory": "$SRC",
+    "replacements": [
+      {
+       "src": "translit/Zawgyi_my.txt",
+       "dest": "translit/Zawgyi_my.txt"
+      },
+      "misc/dayPeriods.txt"
+    ]
+  }
 }

diff --git a/docs/userguide/icu_data/buildtool.md b/docs/userguide/icu_data/buildtool.md
index b0930bdf0a4603b1578a5db5723dd43e5ee303db..da4c859fa7ec5cde8a0bfc58752cf8af853583d7 100644 (file)
--- a/docs/userguide/icu_data/buildtool.md
+++ b/docs/userguide/icu_data/buildtool.md
@@ -702,8 +702,21 @@ configuration file:
         {
           src: "Zawgyi_my.txt"
           dest: "translit/Zawgyi_my.txt"
-        }
+        },
+        "misc/dayPeriods.txt"
       ]
     }
 
+`directory` should either be an absolute path, or a path starting with one of
+the following, and it should not contain a trailing slash:
+
+- "$SRC" for the *icu4c/source/data* directory in the source tree
+- "$FILTERS" for the directory containing filters.json
+- "$CWD" for your current working directory
+
+When the entry in the `replacements` array is an object, the `src` and `dest`
+fields indicate, for each file in the source directory (`src`), what file in
+the ICU hierarchy it should replace (`dest`). When the entry is a string, the
+same relative path is used for both `src` and `dest`.
+
 Whole-file substitution happens before all other filters are applied.

diff --git a/icu4c/source/python/icutools/databuilder/__main__.py b/icu4c/source/python/icutools/databuilder/__main__.py
index 7cfec1453bef97c3a7365ca6e7c1f84b4e8a8795..5d843d9c0818b140393f9f3a902ff8361d668be6 100644 (file)
--- a/icu4c/source/python/icutools/databuilder/__main__.py
+++ b/icu4c/source/python/icutools/databuilder/__main__.py
@@ -130,6 +130,7 @@ class Config(object):
 
         # Default fields before processing filter file
         self.filters_json_data = {}
+        self.filter_dir = "ERROR_NO_FILTER_FILE"
 
         # Process filter file
         if args.filter_file:
@@ -140,6 +141,7 @@ class Config(object):
             except IOError:
                 print("Error: Could not read filter file %s." % args.filter_file, file=sys.stderr)
                 exit(1)
+            self.filter_dir = os.path.abspath(os.path.dirname(args.filter_file))
 
         # Either "unihan" or "implicithan"
         self.coll_han_type = "unihan"
@@ -188,7 +190,14 @@ class Config(object):
 def add_copy_input_requests(requests, config, common_vars):
     files_to_copy = set()
     for request in requests:
-        for f in request.all_input_files():
+        request_files = request.all_input_files()
+        # Also add known dependency txt files as possible inputs.
+        # This is required for translit rule files.
+        if hasattr(request, "dep_targets"):
+            request_files += [
+                f for f in request.dep_targets if isinstance(f, InFile)
+            ]
+        for f in request_files:
             if isinstance(f, InFile):
                 files_to_copy.add(f)
 
@@ -198,8 +207,12 @@ def add_copy_input_requests(requests, config, common_vars):
     json_data = config.filters_json_data["fileReplacements"]
     dirname = json_data["directory"]
     for directive in json_data["replacements"]:
-        input_file = LocalFile(dirname, directive["src"])
-        output_file = InFile(directive["dest"])
+        if type(directive) == str:
+            input_file = LocalFile(dirname, directive)
+            output_file = InFile(directive)
+        else:
+            input_file = LocalFile(dirname, directive["src"])
+            output_file = InFile(directive["dest"])
         result += [
             CopyRequest(
                 name = "input_copy_%d" % id,
@@ -240,7 +253,10 @@ def main(argv):
             for key in list(makefile_vars.keys()) + makefile_env
         }
         common["GLOB_DIR"] = args.src_dir
+        common["FILTERS_DIR"] = config.filter_dir
+        common["CWD_DIR"] = os.getcwd()
     else:
+        makefile_vars = None
         common = {
             # GLOB_DIR is used now, whereas IN_DIR is used during execution phase.
             # There is no useful distinction in unix-exec or windows-exec mode.
@@ -249,6 +265,8 @@ def main(argv):
             "IN_DIR": args.src_dir,
             "OUT_DIR": args.out_dir,
             "TMP_DIR": args.tmp_dir,
+            "FILTERS_DIR": config.filter_dir,
+            "CWD_DIR": os.getcwd(),
             "INDEX_NAME": "res_index",
             # TODO: Pull this from configure script:
             "ICUDATA_CHAR": "l"
@@ -271,8 +289,6 @@ def main(argv):
         sys.exit(1)
 
     requests = BUILDRULES.generate(config, glob, common)
-    requests = filtration.apply_filters(requests, config)
-    requests = utils.flatten_requests(requests, config, common)
 
     if "fileReplacements" in config.filters_json_data:
         tmp_in_dir = "{TMP_DIR}/in".format(**common)
@@ -282,6 +298,9 @@ def main(argv):
             common["IN_DIR"] = tmp_in_dir
         requests = add_copy_input_requests(requests, config, common)
 
+    requests = filtration.apply_filters(requests, config)
+    requests = utils.flatten_requests(requests, config, common)
+
     build_dirs = utils.compute_directories(requests)
 
     if args.mode == "gnumake":

diff --git a/icu4c/source/python/icutools/databuilder/filtration_schema.json b/icu4c/source/python/icutools/databuilder/filtration_schema.json
index ba027edce1141be64737462e8b684f3523f1e6a3..2b7ff9989992a12e77a81ad9d6f2a6418351f22e 100644 (file)
--- a/icu4c/source/python/icutools/databuilder/filtration_schema.json
+++ b/icu4c/source/python/icutools/databuilder/filtration_schema.json
@@ -49,17 +49,25 @@
         "fileReplacements": {
             "type": "object",
             "properties": {
-                "directory": { "type": "string" },
+                "directory": {
+                    "type": "string",
+                    "pattern": "^(\\$SRC|\\$FILTERS|\\$CWD|/$|/[^/]+)(/[^/]+)*$"
+                },
                 "replacements": {
                     "type": "array",
                     "items": {
-                        "type": "object",
-                        "properties": {
-                            "src": { "type": "string" },
-                            "dest": { "type": "string" }
-                        },
-                        "additionalProperties": false,
-                        "required": ["src", "dest"]
+                        "oneOf": [
+                            { "type": "string" },
+                            {
+                                "type": "object",
+                                "properties": {
+                                    "src": { "type": "string" },
+                                    "dest": { "type": "string" }
+                                },
+                                "additionalProperties": false,
+                                "required": ["src", "dest"]
+                            }
+                        ]
                     }
                 }
             },

diff --git a/icu4c/source/python/icutools/databuilder/request_types.py b/icu4c/source/python/icutools/databuilder/request_types.py
index 9a973faec1bd97303f9480d789d612588257a178..d7ffd799d19cd9ee84411f905faa848ace380337 100644 (file)
--- a/icu4c/source/python/icutools/databuilder/request_types.py
+++ b/icu4c/source/python/icutools/databuilder/request_types.py
@@ -147,6 +147,7 @@ class AbstractExecutionRequest(AbstractRequest):
                     dep_target.name,
                     self.name
                 ), file=sys.stderr)
+        self.dep_targets = []
 
     def all_input_files(self):
         return self.common_dep_files + self.input_files

diff --git a/icu4c/source/python/icutools/databuilder/utils.py b/icu4c/source/python/icutools/databuilder/utils.py
index 196a79967017676ce22436f3baa2a48b9d433910..c5965906c6b2f1d1f6ff205fe229c502e4665676 100644 (file)
--- a/icu4c/source/python/icutools/databuilder/utils.py
+++ b/icu4c/source/python/icutools/databuilder/utils.py
@@ -12,7 +12,7 @@ from . import *
 
 def dir_for(file):
     if isinstance(file, LocalFile):
-        return file.dirname
+        return get_local_dirname(file.dirname)
     if isinstance(file, SrcFile):
         return "{SRC_DIR}"
     if isinstance(file, InFile):
@@ -26,6 +26,32 @@ def dir_for(file):
     assert False
 
 
+LOCAL_DIRNAME_SUBSTITUTIONS = {
+    "SRC": "{SRC_DIR}",
+    "FILTERS": "{FILTERS_DIR}",
+    "CWD": "{CWD_DIR}"
+}
+
+
+def get_local_dirname(dirname):
+    if dirname.startswith("/"):
+        return dirname
+    elif dirname.startswith("$"):
+        # Note: directory separator substitution happens later
+        sep_idx = dirname.find("/")
+        if sep_idx == -1:
+            sep_idx = len(dirname)
+        variable = dirname[1:sep_idx]
+        if variable in LOCAL_DIRNAME_SUBSTITUTIONS:
+            return LOCAL_DIRNAME_SUBSTITUTIONS[variable] + dirname[sep_idx:]
+    print(
+        "Error: Local directory must be absolute, or relative to one of: " +
+        (", ".join("$%s" % v for v in LOCAL_DIRNAME_SUBSTITUTIONS.keys())),
+        file=sys.stderr
+    )
+    exit(1)
+
+
 def concat_dicts(*dicts):
     # There is not a super great way to do this in Python:
     new_dict = {}