+++ /dev/null
-/*
- *******************************************************************************
- * Copyright (C) 1996-2009, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- *******************************************************************************
- */
-package com.ibm.icu.dev.tool.rbbi;
-
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
-import java.util.Vector;
-
-import com.ibm.icu.text.UnicodeSet;
-import com.ibm.icu.util.CompactByteArray;
-
-public class BuildDictionaryFile {
- public static void main(String args[])
- throws FileNotFoundException, UnsupportedEncodingException, IOException {
- String filename = args[0];
- String encoding = "";
- String outputFile = "";
- String listingFile = "";
-
- if (args.length >= 2)
- encoding = args[1];
-
- if(args.length >= 3)
- outputFile = args[2];
-
- if (args.length >= 4)
- listingFile = args[3];
-
- BuildDictionaryFile dictionary = new BuildDictionaryFile();
- dictionary.build(filename, encoding);
-
- DataOutputStream out = null;
- if (outputFile.length() != 0) {
- out = new DataOutputStream(new FileOutputStream(outputFile));
- dictionary.writeDictionaryFile(out);
- }
-
- PrintWriter listing = null;
- if (listingFile.length() != 0) {
- listing = new PrintWriter(new OutputStreamWriter(new FileOutputStream(listingFile), "UnicodeLittle"));
- dictionary.printWordList("", 0, listing);
- listing.close();
- }
- }
-
- public BuildDictionaryFile() {
- }
-
- public void build(String filename, String encoding)
- throws FileNotFoundException, UnsupportedEncodingException, IOException {
- FileInputStream file = new FileInputStream(filename);
- InputStreamReader in;
- if (encoding.length() == 0)
- in = new InputStreamReader(file);
- else
- in = new InputStreamReader(file, encoding);
-
- buildColumnMap(in);
-
- file = new FileInputStream(filename);
- if (encoding.length() == 0)
- in = new InputStreamReader(file);
- else
- in = new InputStreamReader(file, encoding);
-
- buildStateTable(in);
-//printTable();
- }
-
- public void buildColumnMap(InputStreamReader in) throws IOException {
-System.out.println("Building column map...");
- UnicodeSet charsInFile = new UnicodeSet();
- int c = in.read();
-int totalChars = 0;
- while (c >= 0) {
-++totalChars; if (totalChars > 0 && totalChars % 5000 == 0) System.out.println("Read " + totalChars + " characters...");
- if (c > ' ')
- charsInFile.add((char)c);
- c = in.read();
- }
-// Test.debugPrintln(charsInFile.toString());
-
- StringBuffer tempReverseMap = new StringBuffer();
- tempReverseMap.append(' ');
-
- columnMap = new CompactByteArray();
- int n = charsInFile.getRangeCount();
- byte p = 1;
- for (int i=0; i<n; ++i) {
- char start = (char) charsInFile.getRangeStart(i);
- char end = (char) charsInFile.getRangeEnd(i);
- for (char ch = start; ch <= end; ch++) {
- if (columnMap.elementAt(Character.toLowerCase(ch)) == 0) {
- columnMap.setElementAt(Character.toUpperCase(ch), Character.toUpperCase(ch),
- p);
- columnMap.setElementAt(Character.toLowerCase(ch), Character.toLowerCase(ch),
- p);
- ++p;
- tempReverseMap.append(ch);
- }
- }
- }
-//System.out.println("Compacting...");
- columnMap.compact();
-
-//System.out.println(tempReverseMap.toString());
- reverseColumnMap = new char[p];
- if (0 != p) {
- tempReverseMap.getChars(0, p, reverseColumnMap, 0);
- }
-
- System.out.println("total columns = " + p);
- numCols = p;
- numColGroups = (numCols >> 5) + 1;
-
-/*
-short[] index = columnMap.getIndexArray();
-System.out.println("Index:");
-for (int i = 0; i < index.length; i++) {
-if (i % 16 == 0) {
-System.out.println();
-System.out.print(" " + Integer.toHexString(i * 128) + ":");
-}
-System.out.print("\t" + Integer.toHexString(index[i]));
-}
-System.out.println();
-byte[] data = columnMap.getStringArray();
-System.out.print("Values:");
-for (int i = 0; i < data.length; i++) {
-if (i % 16 == 0) {
-System.out.println();
-System.out.print(" " + Integer.toHexString(i) + ":");
-}
-if (data[i] == 0)
-System.out.print("\t.");
-else
-System.out.print("\t" + Integer.toString(data[i]));
-}
-System.out.println();
-*/
- }
-
- public void buildStateTable(InputStreamReader in) throws IOException {
- Vector tempTable = new Vector();
- tempTable.addElement(new int[numCols + 1]);
- int state = 0;
-
- int c = in.read();
- int[] row = null;
- int charsInWord = 0;
- while (c >= 0) {
- charsInWord++;
- short column = columnMap.elementAt((char)c);
-
- row = (int[])(tempTable.elementAt(state));
- if (column != 0) {
- if (row[column] == 0) {
- row[column] = tempTable.size();
- ++row[numCols];
- state = (tempTable.size());
- tempTable.addElement(new int[numCols + 1]);
- }
- else
- state = row[column];
- }
- else if (state != 0) {
- if (row[0] != -1) {
- row[0] = -1;
- ++row[numCols];
- uniqueWords++;
- totalUniqueWordChars += charsInWord;
- }
- totalWords++;
-if (totalWords % 5000 == 0) System.out.println("Read " + totalWords + " words, " + tempTable.size() + " rows...");
- charsInWord = 0;
- state = 0;
- }
- c = in.read();
- }
- if (state != 0) {
- row = (int[])(tempTable.elementAt(state));
- if (row[0] != -1) {
- row[0] = -1;
- uniqueWords++;
- totalUniqueWordChars += charsInWord;
- }
- totalWords++;
- }
-
- compress(tempTable);
-
- table = new short[numCols * tempTable.size()];
- for (int i = 0; i < tempTable.size(); i++) {
- row = (int[])tempTable.elementAt(i);
- for (int j = 0; j < numCols; j++)
- table[i * numCols + j] = (short)row[j];
- }
- }
-
- private void compress(Vector tempTable) {
-System.out.println("Before compression:");
-System.out.println(" Number of rows = " + tempTable.size());
-System.out.println(" Number of columns = " + numCols);
-System.out.println(" Number of cells = " + tempTable.size() * numCols);
- deleteDuplicateRows(tempTable);
-System.out.println("After removing duplicate rows:");
-System.out.println(" Number of rows = " + tempTable.size());
-System.out.println(" Number of columns = " + numCols);
-System.out.println(" Number of cells = " + tempTable.size() * numCols);
- stackRows(tempTable);
-if (tempTable.size() > 32767) throw new IllegalArgumentException("Too many rows in table!");
-System.out.println("After doubling up on rows:");
-System.out.println(" Number of rows = " + tempTable.size());
-System.out.println(" Number of columns = " + numCols);
-System.out.println(" Number of cells = " + tempTable.size() * numCols);
- }
-
-/*
-experimental...
- private void deleteDuplicateRows(Vector tempTable) {
- int[] rowNumMap = new int[tempTable.size()];
- for (int i = 0; i < rowNumMap.length; i++)
- rowNumMap[i] = i;
-
- int nextClass = numCols;
- int currentClass;
- int lastClass;
- boolean split;
- int[] row1, row2, tempRow;
- int tempCat;
-
- do {
-System.out.println("Making a pass (" + nextClass + " classes)...");
- currentClass = 0;
- lastClass = nextClass;
- while (currentClass < nextClass) {
-System.out.println(" currentClass = " + currentClass);
- split = false;
- row1 = row2 = null;
- for (int i = 0; i < tempTable.size(); i++) {
- tempRow = (int[])tempTable.elementAt(i);
- if (tempRow[numCols] == currentClass) {
- if (row1 == null) {
- row1 = (int[])tempTable.elementAt(i);
- }
- else {
- row2 = (int[])tempTable.elementAt(i);
- for (int j = 0; j < numCols; j++) {
- if ((row1[j] == 0) != (row2[j] == 0) ||
- (row1[j] == -1) != (row2[j] == -1)) {
- row2[numCols] = nextClass;
- split = true;
- break;
- }
- else if (row1[j] != 0 && row2[j] != 0 && row1[j] != -1
- && row2[j] != -1) {
- tempRow = (int[])tempTable.elementAt(row1[j]);
- tempCat = tempRow[numCols];
- tempRow = (int[])tempTable.elementAt(row2[j]);
- if (tempCat != tempRow[numCols]) {
- row2[numCols] = nextClass;
- split = true;
- break;
- }
- }
- }
- }
- }
- }
- if (split)
- ++nextClass;
- ++currentClass;
-//System.out.println();
- }
- } while (lastClass != nextClass);
-
- int[] representatives = new int[nextClass];
- for (int i = 1; i < tempTable.size(); i++) {
- tempRow = (int[])tempTable.elementAt(i);
- if (representatives[tempRow[numCols]] == 0)
- representatives[tempRow[numCols]] = i;
- else
- rowNumMap[i] = representatives[tempRow[numCols]];
- }
-System.out.println("Renumbering...");
-
- // renumber all remaining rows
- for (int i = 0; i < rowNumMap.length; i++)
- if (rowNumMap[i] != i)
- tempTable.setElementAt(null, i);
- int newRowNum = 0;
- for (int i = 0; i < rowNumMap.length; i++)
- if (tempTable.elementAt(i) != null)
- rowNumMap[i] = newRowNum++;
- for (int i = 0; i < rowNumMap.length; i++)
- if (tempTable.elementAt(i) == null)
- rowNumMap[i] = rowNumMap[rowNumMap[i]];
-
- for (int i = tempTable.size() - 1; i >= 0; i--) {
- tempRow = (int[])tempTable.elementAt(i);
- if (tempRow == null)
- tempTable.removeElementAt(i);
- else {
- for (int j = 0; j < numCols; j++)
- if (tempRow[j] != -1)
- tempRow[j] = rowNumMap[j];
- }
- }
-//for (int i = 1; i < rowNumMap.length; i++) rowNumMap[i] = i; int newRowNum = rowNumMap.length;
- }
-*/
-
- private void deleteDuplicateRows(Vector tempTable) {
- Vector work = (Vector)(tempTable.clone());
- boolean didDeleteRow = true;
-
- Vector tempMapping = new Vector(work.size());
- int[] mapping = new int[work.size()];
- for (int i = 0; i < mapping.length; i++) {
- mapping[i] = i;
- tempMapping.addElement(new Integer(i));
- }
- boolean[] tbd = new boolean[work.size()];
-
- while (didDeleteRow) {
-System.out.println(" " + work.size() + " rows...");
-int deletedRows = 0;
- didDeleteRow = false;
-
- sortTable(work, tempMapping, mapping, 1, work.size());
- for (int i = 0; i < work.size() - 1; ) {
-System.out.print("Deleting, inspecting row " + i + ", deleted " + deletedRows + " rows...\r");
- int rowToDelete = ((Integer)(tempMapping.elementAt(i + 1))).intValue();
- int rowToMapTo = ((Integer)(tempMapping.elementAt(i))).intValue();
- if (compareRows((int[])work.elementAt(i), (int[])work.elementAt(i + 1),
- mapping) == 0) {
- tbd[rowToDelete] = true;
- tempTable.setElementAt(null, rowToDelete);
- while (tbd[mapping[rowToMapTo]])
- mapping[rowToMapTo] = mapping[mapping[rowToMapTo]];
- mapping[rowToDelete] = mapping[rowToMapTo];
- didDeleteRow = true;
-deletedRows++;
- work.removeElementAt(i + 1);
- tempMapping.removeElementAt(i + 1);
- }
- else
- i++;
- }
- for (int i = 0; i < mapping.length; i++) {
- if (tbd[i] && tbd[mapping[i]])
- mapping[i] = mapping[mapping[i]];
- }
- }
-
- int decrementBy = 0;
- for (int i = 0; i < mapping.length; i++) {
- if (tbd[i])
- decrementBy++;
- else
- mapping[i] -= decrementBy;
- }
- for (int i = 0; i < mapping.length; i++) {
- if (tbd[i])
- mapping[i] = mapping[mapping[i]];
- }
- for (int i = tempTable.size() - 1; i >= 0; i--) {
- if (tbd[i])
- tempTable.removeElementAt(i);
- else {
- int[] row = (int[])tempTable.elementAt(i);
- for (int j = 0; j < numCols; j++)
- row[j] = (row[j] == -1) ? -1 : mapping[row[j]];
- }
- }
- }
-
- private void sortTable(Vector tbl, Vector tempMapping, int[] mapping, int start, int end) {
-System.out.print("Sorting (" + start + ", " + end + ")...\r");
- if (start + 1 >= end)
- return;
- else if (start + 10 >= end) {
- for (int i = start + 1; i < end; i++) {
- int[] row = (int[])tbl.elementAt(i);
- Integer tempMap = (Integer)tempMapping.elementAt(i);
- int j;
- for (j = i - 1; j >= start; j--) {
- if (compareRows((int[])tbl.elementAt(j), row, mapping) > 0) {
- tbl.setElementAt((int[])tbl.elementAt(j), j + 1);
- tempMapping.setElementAt((Integer)tempMapping.elementAt(j), j + 1);
- }
- else {
- tbl.setElementAt(row, j + 1);
- tempMapping.setElementAt(tempMap, j + 1);
- break;
- }
- }
- if (j < start) {
- tbl.setElementAt(row, start);
- tempMapping.setElementAt(tempMap, start);
- }
- }
- }
- else {
- int boundaryPos = (start + end) / 2;
- int i;
- boolean allTheSame = true;
- int firstDifferent = 0;
-
- do {
- int[] boundary = (int[])tbl.elementAt(boundaryPos);
- i = start;
- int j = end - 1;
- int[] row = null;
- byte compResult;
- while (i < j) {
- row = (int[])tbl.elementAt(i);
- while (i <= j && compareRows(row, boundary, mapping) < 0) {
- i++;
- row = (int[])tbl.elementAt(i);
- }
-
- row = (int[])tbl.elementAt(j);
- compResult = compareRows(row, boundary, mapping);
- while (i <= j && (compResult >= 0)) {
- if (compResult != 0) {
- allTheSame = false;
- firstDifferent = j;
- }
- j--;
- row = (int[])tbl.elementAt(j);
- compResult = compareRows(row, boundary, mapping);
- }
- if (i <= j) {
- row = (int[])tbl.elementAt(j);
- tbl.setElementAt(tbl.elementAt(i), j);
- tbl.setElementAt(row, i);
- Object temp = tempMapping.elementAt(j);
- tempMapping.setElementAt(tempMapping.elementAt(i), j);
- tempMapping.setElementAt(temp, i);
- }
- }
- if (i <= start) {
- if (allTheSame)
- return;
- else
- boundaryPos = firstDifferent;
- }
- } while (i <= start);
- sortTable(tbl, tempMapping, mapping, start, i);
- sortTable(tbl, tempMapping, mapping, i, end);
- }
- }
-
- private byte compareRows(int[] row1, int[] row2, int[] mapping) {
- for (int i = 0; i < numCols; i++) {
- int c1 = (row1[i] == -1) ? -1 : mapping[row1[i]];
- int c2 = (row2[i] == -1) ? -1 : mapping[row2[i]];
- if (c1 < c2)
- return -1;
- else if (c1 > c2)
- return 1;
- }
- return 0;
- }
-
- private int[] buildRowIndex(Vector tempTable) {
- int[] tempRowIndex = new int[tempTable.size()];
- rowIndexFlagsIndex = new short[tempTable.size()];
- Vector tempRowIndexFlags = new Vector();
- rowIndexShifts = new byte[tempTable.size()];
-
- // build the row index. Each entry in the row index starts out referring
- // to the original row (so it doesn't actually do any mapping), and we set
- // up the index flags to show which cells in the row are populated
- for (int i = 0; i < tempTable.size(); i++) {
- tempRowIndex[i] = i;
-
- int[] row = (int[])tempTable.elementAt(i);
- if (row[numCols] == 1 && row[0] == 0) {
- int j = 0;
- while (row[j] == 0)
- ++j;
- rowIndexFlagsIndex[i] = (short)(-j);
- }
- else {
- int[] flags = new int[numColGroups];
- int nextFlag = 1;
- int colGroup = 0;
- for (int j = 0; j < numCols; j++) {
- if (row[j] != 0)
- flags[colGroup] |= nextFlag;
- nextFlag <<= 1;
- if (nextFlag == 0) {
- ++colGroup;
- nextFlag = 1;
- }
- }
- colGroup = 0;
- int j = 0;
- while (j < tempRowIndexFlags.size()) {
- if (((Integer)tempRowIndexFlags.elementAt(j)).intValue() ==
- flags[colGroup]) {
- ++colGroup;
- ++j;
- if (colGroup >= numColGroups)
- break;
- }
- else if (colGroup != 0)
- colGroup = 0;
- else
- ++j;
- }
- rowIndexFlagsIndex[i] = (short)(j - colGroup);
- while (colGroup < numColGroups) {
- tempRowIndexFlags.addElement(new Integer(flags[colGroup]));
- ++colGroup;
- }
- }
- }
- rowIndexFlags = new int[tempRowIndexFlags.size()];
- for (int i = 0; i < rowIndexFlags.length; i++)
- rowIndexFlags[i] = ((Integer)tempRowIndexFlags.elementAt(i)).intValue();
-System.out.println("Number of column groups = " + numColGroups);
-System.out.println("Size of rowIndexFlags = " + rowIndexFlags.length);
-
- return tempRowIndex;
- }
-
- private void stackRows(Vector tempTable) {
-/*
-System.out.print("Row:\t");
-for (int i = 0; i < numCols; i++)
-System.out.print(reverseColumnMap[i] + "\t");
-System.out.println();
-for (int i = 0; i < tempTable.size(); i++) {
-System.out.print(Integer.toString(i) + ":\t");
-int[] row = (int[])tempTable.elementAt(i);
-for (int j = 0; j < row.length; j++)
-if (row[j] != 0) System.out.print(Integer.toString(row[j]) + "\t");
-else System.out.print(".\t");
-System.out.println();
-}
-*/
-
- int[] tempRowIndex = buildRowIndex(tempTable);
- boolean[] tbd = new boolean[tempTable.size()];
-
- // now we actually go through and stack rows together
- for (int i = 0; i < tempTable.size(); i++) {
- if (tbd[i])
- continue;
-System.out.print("Stacking, inspecting row " + i + "...\r");
-//System.out.println("Stacking, inspecting row " + i + "...");
-
- int[] destRow = (int[])tempTable.elementAt(i);
-
- boolean[] tempFlags = new boolean[numCols];
- boolean[] filledCells = new boolean[numCols];
- for (int j = 0; j < numCols; j++)
- filledCells[j] = destRow[j] != 0;
-
- for (int j = i + 1; destRow[numCols] < numCols && j < tempTable.size(); j++) {
- if (tbd[j])
- continue;
-
- int[] srcRow = (int[])tempTable.elementAt(j);
- if (srcRow[numCols] + destRow[numCols] > numCols)
- continue;
-
- int maxLeftShift = -999;
- int maxRightShift = 0;
- for (int k = 0; k < numCols; k++) {
- tempFlags[k] = srcRow[k] != 0;
- if (tempFlags[k]) {
- if (maxLeftShift == -999)
- maxLeftShift = -k;
- maxRightShift = (numCols - 1) - k;
- }
- }
-
- int shift;
- for (shift = maxLeftShift; shift <= maxRightShift; shift++) {
- int k;
- for (k = 0; k < numCols; k++) {
- if (tempFlags[k] && filledCells[k + shift])
- break;
- }
- if (k >= numCols)
- break;
- }
- if (shift <= maxRightShift) {
-//System.out.println("Packing row " + j + " into row " + i + " with shift = " + shift);
- for (int k = 0; k < numCols; k++) {
- if (tempFlags[k]) {
- filledCells[k + shift] = true;
- destRow[k + shift] = srcRow[k];
- ++destRow[numCols];
- }
- }
- tbd[j] = true;
- tempRowIndex[j] = i;
- rowIndexShifts[j] = (byte)shift;
- }
- }
- }
-
- // finally, we squeeze out all the deleted rows
- int decrementBy = 0;
- for (int i = 0; i < tempRowIndex.length; i++) {
- if (!tbd[i])
- tempRowIndex[i] -= decrementBy;
- else
- ++decrementBy;
- }
- rowIndex = new short[tempRowIndex.length];
- for (int i = tempRowIndex.length - 1; i >= 0; i--) {
- if (tbd[i]) {
- rowIndex[i] = (short)(tempRowIndex[tempRowIndex[i]]);
- tempTable.removeElementAt(i);
- }
- else
- rowIndex[i] = (short)tempRowIndex[i];
- }
- }
-
-// private void printTable() {
-// short cell;
-// int populatedCells = 0;
-///*
-// System.out.println("Conceptual table:");
-// System.out.println(" Row: a b c d e f g h i j k l m n"
-// + " o p q r s t u v w x y z ' #");
-//
-// boolean[] rowPrintFlags = new boolean[rowIndex.length];
-// printConceptualTable("", 0, rowPrintFlags);
-//*/
-//
-// System.out.println();
-// System.out.println("Conceptual table:");
-// System.out.print(" Row:");
-// for (int i = 0; i < reverseColumnMap.length; i++) {
-// System.out.print(" " + reverseColumnMap[i]);
-// }
-// for (int i = 0; i < rowIndex.length; i++) {
-// System.out.println();
-// printNumber(i, 4);
-// System.out.print(":");
-// for (int j = 0; j < numCols; j++)
-// printNumber(at(i, j), 4);
-// }
-// System.out.println('\n');
-//
-// System.out.println();
-// System.out.println("Internally stored table:");
-// System.out.print(" Row:");
-// for (int i = 0; i < reverseColumnMap.length; i++) {
-// System.out.print(" " + reverseColumnMap[i]);
-// }
-// for (int i = 0; i < table.length; i++) {
-// if (i % numCols == 0) {
-// System.out.println();
-// printNumber(i / numCols, 4);
-// System.out.print(":");
-// }
-// cell = table[i];
-// if (cell != 0)
-// populatedCells++;
-// printNumber(cell, 4);
-// }
-// System.out.println('\n');
-//
-//System.out.println("Row index:");
-//for (int i = 0; i < rowIndex.length; i++) {
-// System.out.print(" " + i + " -> " + rowIndex[i]);
-// if (rowIndexFlagsIndex[i] < 0)
-// System.out.print(", flags = " + Integer.toBinaryString((1 << (-rowIndexFlagsIndex[i]))) + " (" + rowIndexFlagsIndex[i]);
-// else
-// System.out.print(", flags = " + Integer.toBinaryString(rowIndexFlags[rowIndexFlagsIndex[i]]) + " (" + rowIndexFlagsIndex[i]);
-// System.out.println("), shift = " + rowIndexShifts[i]);
-//}
-///*
-// int theoreticalMinRows = populatedCells / numCols;
-// if (populatedCells % numCols != 0)
-// theoreticalMinRows++;
-// int oneCellRows = 0;
-// for (int i = 0; i < rowIndexFlags.length; i++) {
-// double temp = Math.log(rowIndexFlags[i]) / Math.log(2);
-// if (temp == (int)temp)
-// oneCellRows++;
-// }
-//
-// System.out.println('\n');
-// System.out.println("Total words in input = " + totalWords);
-// System.out.println("Total unique words = " + uniqueWords + ", comprising " +
-// totalUniqueWordChars + " characters\n");
-// System.out.println("Number of populated cells = " + populatedCells);
-// System.out.println("Total number of cells = " + (table.length));
-// System.out.println("Residency = " + ((float)populatedCells / table.length * 100) + '%');
-// System.out.println("Ratio of populated cells to unique-word characters = " +
-// ((float)populatedCells / totalUniqueWordChars * 100) + '%');
-// System.out.println("Ratio of total cells to unique-word characters = " +
-// ((float)table.length / totalUniqueWordChars * 100) + '%');
-// System.out.println("Number of rows = " + (table.length / numCols));
-// System.out.println("Theoretical minimum number of rows = " + theoreticalMinRows);
-// System.out.println("Ratio of number of rows to theoretical minimum = " +
-// ((float)(table.length / numCols) / theoreticalMinRows * 100) + '%');
-// System.out.println("Number of conceptual rows = " + rowIndex.length);
-// System.out.println("Conceptual rows with only one populated cell = " + oneCellRows);
-// System.out.println("Ratio of one-cell rows to total conceptual rows = " + (((float)oneCellRows)
-// / rowIndex.length * 100) + '%');
-// System.out.println("Average number of populated cells in multi-cell rows = " +
-// ((float)(populatedCells - oneCellRows) / (rowIndex.length - oneCellRows)));
-//
-// int storageUsed = table.length * 2 + rowIndex.length * 2
-// + rowIndexFlags.length * 4 + rowIndexShifts.length;
-// System.out.println("Total number of bytes in table (including indexes) = " +
-// storageUsed);
-// System.out.println("Bytes of overhead per unique-word character = " + ((double)(storageUsed
-// - (totalUniqueWordChars * 2)) / totalUniqueWordChars));
-//*/
-// }
-
-// private void printConceptualTable(String initialString, int state, boolean[] flags) {
-// if (initialString.length() == 0)
-// System.out.println("root:");
-// else
-// System.out.println(initialString + ':');
-//
-// if (!flags[state]) {
-// flags[state] = true;
-// printNumber(state, 4);
-// System.out.print(":");
-// for (int i = 0; i < numCols; i++)
-// printNumber(at(state, i), 4);
-// System.out.println();
-// }
-//
-// int nextState;
-// for (int i = 0; i < numCols; i++) {
-// nextState = at(state, i);
-// if (nextState > 0 && !flags[nextState]) {
-// printNumber(nextState, 4);
-// System.out.print(":");
-// for (int j = 0; j < numCols; j++)
-// printNumber(at(nextState, j), 4);
-// System.out.println();
-// }
-// }
-// for (int i = 0; i < numCols; i++) {
-// nextState = at(state, i);
-// if (nextState > 0 && !flags[nextState]) {
-// char nextChar;
-// if (nextState == 27)
-// nextChar = ' ';
-// else if (nextState == 26)
-// nextChar = '\'';
-// else
-// nextChar = (char)(i + 'a');
-// flags[nextState] = true;
-// printConceptualTable(initialString + nextChar, nextState, flags);
-// }
-// }
-// }
-
- private void printWordList(String partialWord, int state, PrintWriter out)
- throws IOException {
- if (state == -1) {
- System.out.println(partialWord);
- if (out != null)
- out.println(partialWord);
- }
- else {
- for (int i = 0; i < numCols; i++) {
- if (at(state, i) != 0)
- printWordList(partialWord + reverseColumnMap[i], at(state, i), out);
- }
- }
- }
-
- private void writeDictionaryFile(DataOutputStream out) throws IOException {
- out.writeInt(0); // version number
-
- char[] columnMapIndexes = columnMap.getIndexArray();
- out.writeInt(columnMapIndexes.length);
- for (int i = 0; i < columnMapIndexes.length; i++)
- out.writeShort((short)columnMapIndexes[i]);
- byte[] columnMapValues = columnMap.getValueArray();
- out.writeInt(columnMapValues.length);
- for (int i = 0; i < columnMapValues.length; i++)
- out.writeByte((byte)columnMapValues[i]);
-
- out.writeInt(numCols);
- out.writeInt(numColGroups);
-
- out.writeInt(rowIndex.length);
- for (int i = 0; i < rowIndex.length; i++)
- out.writeShort(rowIndex[i]);
-
- out.writeInt(rowIndexFlagsIndex.length);
- for (int i = 0; i < rowIndexFlagsIndex.length; i++)
- out.writeShort(rowIndexFlagsIndex[i]);
- out.writeInt(rowIndexFlags.length);
- for (int i = 0; i < rowIndexFlags.length; i++)
- out.writeInt(rowIndexFlags[i]);
-
- out.writeInt(rowIndexShifts.length);
- for (int i = 0; i < rowIndexShifts.length; i++)
- out.writeByte(rowIndexShifts[i]);
-
- out.writeInt(table.length);
- for (int i = 0; i < table.length; i++)
- out.writeShort(table[i]);
-
- out.close();
- }
-
-// private void printNumber(int x, int width) {
-// String s = String.valueOf(x);
-// if (width > s.length())
-// System.out.print(spaces.substring(0, width - s.length()));
-// if (x != 0)
-// System.out.print(s);
-// else
-// System.out.print('.');
-// }
-
- public final short at(int row, char ch) {
- int col = columnMap.elementAt(ch);
- return at(row, col);
- }
-
- public final short at(int row, int col) {
- if (cellIsPopulated(row, col))
- return internalAt(rowIndex[row], col + rowIndexShifts[row]);
- else
- return 0;
- }
-
- private final boolean cellIsPopulated(int row, int col) {
- if (rowIndexFlagsIndex[row] < 0)
- return col == -rowIndexFlagsIndex[row];
- else {
- int flags = rowIndexFlags[rowIndexFlagsIndex[row] + (col >> 5)];
- return (flags & (1 << (col & 0x1f))) != 0;
- }
- }
-
- private final short internalAt(int row, int col) {
- return table[row * numCols + col];
- }
-
- private CompactByteArray columnMap = null;
- private char[] reverseColumnMap = null;
- private int numCols;
- private int numColGroups;
- private short[] table = null;
- private short[] rowIndex = null;
- private int[] rowIndexFlags = null;
- private short[] rowIndexFlagsIndex = null;
- private byte[] rowIndexShifts = null;
-
- private int totalWords = 0;
- private int uniqueWords = 0;
- private int totalUniqueWordChars = 0;
-
- //private static final String spaces = " ";
-}
-
projects / icu / commitdiff