From: Steven R. Loomis Date: Mon, 13 Mar 2017 23:19:33 +0000 (+0000) Subject: ICU-12766 wip escaping for utf-8 input X-Git-Tag: release-59-rc~99^2~13 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=4890638e080050bdc25acbf1519b956dc3ec84f4;p=icu ICU-12766 wip escaping for utf-8 input * create a new escaper tool - needs to be invoked in mh- files * escapes to temporary _whatever.cpp files * does NOT handle multi line u"xxx" (or u'xxx'? ) * does not cleanup _*.cpp files * fixup a bunch of Makefiles to add -I$(srcdir) (doesn't hurt anyway) X-SVN-Rev: 39787 --- diff --git a/icu4c/source/Makefile.in b/icu4c/source/Makefile.in index bd5b117cb7d..ef2657fc2dc 100644 --- a/icu4c/source/Makefile.in +++ b/icu4c/source/Makefile.in @@ -140,6 +140,7 @@ $(LIBDIR) $(BINDIR): ## Recursive targets all-recursive install-recursive clean-recursive distclean-recursive dist-recursive check-recursive check-exhaustive-recursive: $(LIBDIR) $(BINDIR) + @(cd tools/escapesrc && $(MAKE) RECURSIVE=YES $$local_target) || exit @dot_seen=no; \ target=`echo $@ | sed s/-recursive//`; \ list='$(LOCAL_SUBDIRS)'; for subdir in $$list; do \ diff --git a/icu4c/source/configure b/icu4c/source/configure index 77a6b431c81..b4201bc44b3 100755 --- a/icu4c/source/configure +++ b/icu4c/source/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for ICU 59.1 +# Generated by GNU Autoconf 2.69 for ICU 59.1. # # Report bugs to . # @@ -674,7 +674,6 @@ GENCCODE_ASSEMBLY HAVE_MMAP LIB_THREAD U_HAVE_ATOMIC -U_HAVE_STD_STRING ENABLE_RPATH U_ENABLE_DYLOAD U_HAVE_PLUGINS @@ -5958,37 +5957,6 @@ $as_echo "$as_me: Adding CXXFLAGS option -std=c++11" >&6;} fi fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if #include works" >&5 -$as_echo_n "checking if #include works... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_cxx_try_compile "$LINENO"; then : - ac_cv_header_stdstring=yes -else - ac_cv_header_stdstring=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdstring" >&5 -$as_echo "$ac_cv_header_stdstring" >&6; } -if test $ac_cv_header_stdstring = yes -then - U_HAVE_STD_STRING=1 -else - U_HAVE_STD_STRING=0 - CONFIG_CPPFLAGS="${CONFIG_CPPFLAGS} -DU_HAVE_STD_STRING=0" -fi - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if #include works" >&5 $as_echo_n "checking if #include works... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -7800,7 +7768,7 @@ echo "CXXFLAGS=$CXXFLAGS" # output the Makefiles -ac_config_files="$ac_config_files icudefs.mk Makefile data/pkgdataMakefile config/Makefile.inc config/icu.pc config/pkgdataMakefile data/Makefile stubdata/Makefile common/Makefile i18n/Makefile layoutex/Makefile io/Makefile extra/Makefile extra/uconv/Makefile extra/uconv/pkgdataMakefile extra/scrptrun/Makefile tools/Makefile tools/ctestfw/Makefile tools/toolutil/Makefile tools/makeconv/Makefile tools/genrb/Makefile tools/genccode/Makefile tools/gencmn/Makefile tools/gencnval/Makefile tools/gendict/Makefile tools/gentest/Makefile tools/gennorm2/Makefile tools/genbrk/Makefile tools/gensprep/Makefile tools/icuinfo/Makefile tools/icupkg/Makefile tools/icuswap/Makefile tools/pkgdata/Makefile tools/tzcode/Makefile tools/gencfu/Makefile test/Makefile test/compat/Makefile test/testdata/Makefile test/testdata/pkgdataMakefile test/hdrtst/Makefile test/intltest/Makefile test/cintltst/Makefile test/iotest/Makefile test/letest/Makefile test/perf/Makefile test/perf/collationperf/Makefile test/perf/collperf/Makefile test/perf/collperf2/Makefile test/perf/dicttrieperf/Makefile test/perf/ubrkperf/Makefile test/perf/charperf/Makefile test/perf/convperf/Makefile test/perf/normperf/Makefile test/perf/DateFmtPerf/Makefile test/perf/howExpensiveIs/Makefile test/perf/strsrchperf/Makefile test/perf/unisetperf/Makefile test/perf/usetperf/Makefile test/perf/ustrperf/Makefile test/perf/utfperf/Makefile test/perf/utrie2perf/Makefile test/perf/leperf/Makefile samples/Makefile samples/date/Makefile samples/cal/Makefile samples/layout/Makefile" +ac_config_files="$ac_config_files icudefs.mk Makefile data/pkgdataMakefile config/Makefile.inc config/icu.pc config/pkgdataMakefile data/Makefile stubdata/Makefile common/Makefile i18n/Makefile layoutex/Makefile io/Makefile extra/Makefile extra/uconv/Makefile extra/uconv/pkgdataMakefile extra/scrptrun/Makefile tools/Makefile tools/ctestfw/Makefile tools/toolutil/Makefile tools/makeconv/Makefile tools/genrb/Makefile tools/genccode/Makefile tools/gencmn/Makefile tools/gencnval/Makefile tools/gendict/Makefile tools/gentest/Makefile tools/gennorm2/Makefile tools/genbrk/Makefile tools/gensprep/Makefile tools/icuinfo/Makefile tools/icupkg/Makefile tools/icuswap/Makefile tools/pkgdata/Makefile tools/tzcode/Makefile tools/gencfu/Makefile tools/escapesrc/Makefile test/Makefile test/compat/Makefile test/testdata/Makefile test/testdata/pkgdataMakefile test/hdrtst/Makefile test/intltest/Makefile test/cintltst/Makefile test/iotest/Makefile test/letest/Makefile test/perf/Makefile test/perf/collationperf/Makefile test/perf/collperf/Makefile test/perf/collperf2/Makefile test/perf/dicttrieperf/Makefile test/perf/ubrkperf/Makefile test/perf/charperf/Makefile test/perf/convperf/Makefile test/perf/normperf/Makefile test/perf/DateFmtPerf/Makefile test/perf/howExpensiveIs/Makefile test/perf/strsrchperf/Makefile test/perf/unisetperf/Makefile test/perf/usetperf/Makefile test/perf/ustrperf/Makefile test/perf/utfperf/Makefile test/perf/utrie2perf/Makefile test/perf/leperf/Makefile samples/Makefile samples/date/Makefile samples/cal/Makefile samples/layout/Makefile" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure @@ -8545,6 +8513,7 @@ do "tools/pkgdata/Makefile") CONFIG_FILES="$CONFIG_FILES tools/pkgdata/Makefile" ;; "tools/tzcode/Makefile") CONFIG_FILES="$CONFIG_FILES tools/tzcode/Makefile" ;; "tools/gencfu/Makefile") CONFIG_FILES="$CONFIG_FILES tools/gencfu/Makefile" ;; + "tools/escapesrc/Makefile") CONFIG_FILES="$CONFIG_FILES tools/escapesrc/Makefile" ;; "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;; "test/compat/Makefile") CONFIG_FILES="$CONFIG_FILES test/compat/Makefile" ;; "test/testdata/Makefile") CONFIG_FILES="$CONFIG_FILES test/testdata/Makefile" ;; diff --git a/icu4c/source/extra/uconv/Makefile.in b/icu4c/source/extra/uconv/Makefile.in index 7d0af0420bf..b6144619d00 100644 --- a/icu4c/source/extra/uconv/Makefile.in +++ b/icu4c/source/extra/uconv/Makefile.in @@ -44,7 +44,7 @@ CLEANFILES = *~ $(DEPS) $(ALL_MAN_FILES) ## Target information TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) -CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(srcdir)/../toolutil +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(srcdir)/../toolutil CPPFLAGS += -DUNISTR_FROM_CHAR_EXPLICIT=explicit -DUNISTR_FROM_STRING_EXPLICIT=explicit LIBS = $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) diff --git a/icu4c/source/io/Makefile.in b/icu4c/source/io/Makefile.in index 84843254334..45dbe870fb3 100644 --- a/icu4c/source/io/Makefile.in +++ b/icu4c/source/io/Makefile.in @@ -55,7 +55,7 @@ DYNAMICCXXFLAGS = $(SHAREDLIBCXXFLAGS) CFLAGS += $(LIBCFLAGS) CXXFLAGS += $(LIBCXXFLAGS) -CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n $(LIBCPPFLAGS) $(CPPFLAGSICUIO) +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(top_srcdir)/i18n $(LIBCPPFLAGS) $(CPPFLAGSICUIO) DEFS += -DU_IO_IMPLEMENTATION LDFLAGS += $(LDFLAGSICUIO) LIBS = $(LIBICUUC) $(LIBICUI18N) $(DEFAULT_LIBS) diff --git a/icu4c/source/test/intltest/Makefile.in b/icu4c/source/test/intltest/Makefile.in index 073935fcd9f..416d0a91bf9 100644 --- a/icu4c/source/test/intltest/Makefile.in +++ b/icu4c/source/test/intltest/Makefile.in @@ -34,7 +34,7 @@ BUILDDIR := $(BUILDDIR:test\\intltest/../../=) # Simplify the path for Windows 98 BUILDDIR := $(BUILDDIR:TEST\\INTLTEST/../../=) -CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(top_srcdir)/tools/toolutil -I$(top_srcdir)/tools/ctestfw +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(top_srcdir)/tools/toolutil -I$(top_srcdir)/tools/ctestfw CPPFLAGS += -DUNISTR_FROM_CHAR_EXPLICIT= -DUNISTR_FROM_STRING_EXPLICIT= DEFS += -D'U_TOPSRCDIR="$(top_srcdir)/"' -D'U_TOPBUILDDIR="$(BUILDDIR)"' LIBS = $(LIBCTESTFW) $(LIBICUI18N) $(LIBICUUC) $(LIBICUTOOLUTIL) $(DEFAULT_LIBS) $(LIB_M) $(LIB_THREAD) diff --git a/icu4c/source/test/iotest/Makefile.in b/icu4c/source/test/iotest/Makefile.in index 192e4b30eeb..9d2b94c17e4 100644 --- a/icu4c/source/test/iotest/Makefile.in +++ b/icu4c/source/test/iotest/Makefile.in @@ -34,7 +34,7 @@ BUILDDIR := $(BUILDDIR:test\\iotest/../../=) # Simplify the path for Windows 98 BUILDDIR := $(BUILDDIR:TEST\\IOTEST/../../=) -CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(top_srcdir)/tools/ctestfw -I$(top_srcdir)/io +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(top_srcdir)/tools/ctestfw -I$(top_srcdir)/io CPPFLAGS += -DUNISTR_FROM_CHAR_EXPLICIT= -DUNISTR_FROM_STRING_EXPLICIT= DEFS += -D'U_TOPSRCDIR="$(top_srcdir)/"' -D'U_TOPBUILDDIR="$(BUILDDIR)"' LIBS = $(LIBCTESTFW) $(LIBICUTOOLUTIL) $(LIBICUIO) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) diff --git a/icu4c/source/tools/escapesrc/Makefile.in b/icu4c/source/tools/escapesrc/Makefile.in index b1e45505055..71ffdbca07c 100644 --- a/icu4c/source/tools/escapesrc/Makefile.in +++ b/icu4c/source/tools/escapesrc/Makefile.in @@ -5,6 +5,8 @@ ## others. All Rights Reserved. ## Steven R. Loomis +SKIP_ESCAPING=YES + ## Source directory information srcdir = @srcdir@ top_srcdir = @top_srcdir@ diff --git a/icu4c/source/tools/escapesrc/escapesrc.cpp b/icu4c/source/tools/escapesrc/escapesrc.cpp index bcf53d78b92..e4af76f4503 100644 --- a/icu4c/source/tools/escapesrc/escapesrc.cpp +++ b/icu4c/source/tools/escapesrc/escapesrc.cpp @@ -2,8 +2,256 @@ // License & terms of use: http://www.unicode.org/copyright.html #include +#include +#include +#include +#include -int main(int argc, const char *argv[]) { - puts("Hi\n"); +// with caution: +#include "unicode/utf8.h" + +std::string prog; + +void usage() { + fprintf(stderr, "%s: usage: %s infile.cpp outfile.cpp\n", prog.c_str(), prog.c_str()); +} + + +int cleanup(const std::string &outfile) { + const char *outstr = outfile.c_str(); + if(outstr && *outstr) { + int rc = unlink(outstr); + if(rc == 0) { + fprintf(stderr, "%s: deleted %s\n", prog.c_str(), outstr); + return 0; + } else { + if( errno == ENOENT ) { + return 0; // File did not exist - no error. + } else { + perror("unlink"); + return 1; + } + } + } return 0; } + +inline bool hasNonAscii(const char *line, size_t len) { + const unsigned char *uline = reinterpret_cast(line); + for(size_t i=0;i 0x7F) { + return true; + } + } + return false; +} + +inline const char *skipws(const char *p, const char *e) { + for(;p %s\n", prog.c_str(), infile.c_str(), outfile.c_str()); + + FILE *inf = fopen(infile.c_str(), "rb"); + if(!inf) { + fprintf(stderr, "%s: could not open input file %s\n", prog.c_str(), infile.c_str()); + cleanup(outfile); + return 1; + } + + FILE *outf = fopen(outfile.c_str(), "w"); + + if(!outf) { + fprintf(stderr, "%s: could not open output file %s\n", prog.c_str(), outfile.c_str()); + fclose(inf); + return 1; + } + + // TODO: any platform variations of this? + fprintf(outf, "#line 1 \"%s\"\n", infile.c_str()); + + size_t len; + char *line; + int no = 0; + std::string linestr; + while((line = fgetln(inf, &len))!= NULL) { + no++; + linestr.assign(line, len); + if(fixLine(no, linestr)) { + fclose(inf); + fclose(outf); + fprintf(stderr, "%s:%d: Fixup failed by %s\n", infile.c_str(), no, prog.c_str()); + cleanup(outfile); + return 1; + } + len = linestr.size(); // size may have changed. + + if(fwrite(linestr.c_str(), 1, linestr.size(), outf) != len) { + fclose(inf); + fclose(outf); + fprintf(stderr, "%s: short write to %s:%d\n", prog.c_str(), outfile.c_str(), no); + cleanup(outfile); + return 1; + } + } + + fclose(inf); + fclose(outf); + return 0; +} + +int main(int argc, const char *argv[]) { + prog = argv[0]; + + if(argc != 3) { + usage(); + return 1; + } + + std::string infile = argv[1]; + std::string outfile = argv[2]; + + return convert(infile, outfile); +} + + +#include "utf_impl.cpp" diff --git a/icu4c/source/tools/gennorm2/Makefile.in b/icu4c/source/tools/gennorm2/Makefile.in index 228076346a4..729a3d0c905 100644 --- a/icu4c/source/tools/gennorm2/Makefile.in +++ b/icu4c/source/tools/gennorm2/Makefile.in @@ -24,7 +24,7 @@ CLEANFILES = *~ $(DEPS) ## Target information TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) -CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(srcdir)/../toolutil LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) OBJECTS = gennorm2.o n2builder.o diff --git a/icu4c/source/tools/genrb/Makefile.in b/icu4c/source/tools/genrb/Makefile.in index 63cd52bb960..7e2360b64ec 100644 --- a/icu4c/source/tools/genrb/Makefile.in +++ b/icu4c/source/tools/genrb/Makefile.in @@ -33,7 +33,7 @@ TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) # derb depends on icuio @ICUIO_TRUE@DERB = $(BINDIR)/$(DERB_STUB_NAME)$(EXEEXT) -CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(srcdir)/../toolutil -I$(top_srcdir)/io +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(srcdir)/../toolutil -I$(top_srcdir)/io CPPFLAGS += -DUNISTR_FROM_CHAR_EXPLICIT=explicit -DUNISTR_FROM_STRING_EXPLICIT=explicit LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) diff --git a/icu4c/source/tools/icuinfo/icuinfo.cpp b/icu4c/source/tools/icuinfo/icuinfo.cpp index 08bd87b85db..1a1ce5d9e4b 100644 --- a/icu4c/source/tools/icuinfo/icuinfo.cpp +++ b/icu4c/source/tools/icuinfo/icuinfo.cpp @@ -1,5 +1,7 @@ // © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html +char16_t CH = u'•'; + /* ******************************************************************************* * @@ -217,6 +219,8 @@ extern int main(int argc, char* argv[]) { UErrorCode errorCode = U_ZERO_ERROR; UBool didSomething = FALSE; + + printf("U+%lx\n", CH); return 0; /* preset then read command line options */ argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); diff --git a/icu4c/source/tools/makeconv/Makefile.in b/icu4c/source/tools/makeconv/Makefile.in index 35d92fe6d9a..64472838a59 100644 --- a/icu4c/source/tools/makeconv/Makefile.in +++ b/icu4c/source/tools/makeconv/Makefile.in @@ -28,7 +28,7 @@ CLEANFILES = *~ $(DEPS) $(MAN_FILES) ## Target information TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) -CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(srcdir)/../toolutil LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) OBJECTS = makeconv.o ucnvstat.o genmbcs.o gencnvex.o diff --git a/icu4c/source/tools/pkgdata/Makefile.in b/icu4c/source/tools/pkgdata/Makefile.in index 6f08b8e8d30..13e12897e4b 100644 --- a/icu4c/source/tools/pkgdata/Makefile.in +++ b/icu4c/source/tools/pkgdata/Makefile.in @@ -32,7 +32,7 @@ endif ## Target information TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) -CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(srcdir)/../toolutil DEFS += -DUDATA_SO_SUFFIX=\".$(SO)\" -DSTATIC_O=\"$(STATIC_O)\" LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) diff --git a/icu4c/source/tools/toolutil/Makefile.in b/icu4c/source/tools/toolutil/Makefile.in index 097fcf1e1bd..82ca0d8a395 100644 --- a/icu4c/source/tools/toolutil/Makefile.in +++ b/icu4c/source/tools/toolutil/Makefile.in @@ -45,7 +45,7 @@ DYNAMICCXXFLAGS = $(SHAREDLIBCXXFLAGS) CFLAGS += $(LIBCFLAGS) CXXFLAGS += $(LIBCXXFLAGS) -CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n $(LIBCPPFLAGS) +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(top_srcdir)/i18n $(LIBCPPFLAGS) # from icuinfo CPPFLAGS+= "-DU_BUILD=\"@build@\"" "-DU_HOST=\"@host@\"" "-DU_CC=\"@CC@\"" "-DU_CXX=\"@CXX@\""