/*
    * Copyright (C) 2007 University of Cambridge
    *
    * This file is part of Fosstrak (www.fosstrak.org).
    *
    * Fosstrak is free software; you can redistribute it and/or
    * modify it under the terms of the GNU Lesser General Public
    * License version 2.1, as published by the Free Software Foundation.
    *
   * Fosstrak is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
   * Lesser General Public License for more details.
   *
   * You should have received a copy of the GNU Lesser General Public
   * License along with Fosstrak; if not, write to the Free
   * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
   * Boston, MA  02110-1301  USA
   */
  
  package org.fosstrak.tdt;
  
  import java.io.BufferedReader;
  import java.io.FileNotFoundException;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.InputStreamReader;
  import java.math.BigInteger;
  import java.net.MalformedURLException;
  import java.net.URL;
  import java.net.URLConnection;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import javax.xml.bind.JAXBContext;
  import javax.xml.bind.JAXBElement;
  import javax.xml.bind.JAXBException;
  import javax.xml.bind.MarshalException;
  import javax.xml.bind.Unmarshaller;
  import javax.xml.bind.ValidationException;
  import javax.xml.parsers.DocumentBuilder;
  import javax.xml.parsers.DocumentBuilderFactory;
  import javax.xml.parsers.ParserConfigurationException;
  import javax.xml.transform.stream.StreamSource;
  import javax.xml.xpath.XPath;
  import javax.xml.xpath.XPathConstants;
  import javax.xml.xpath.XPathExpressionException;
  import javax.xml.xpath.XPathFactory;
  
  import org.epcglobalinc.tdt.EpcTagDataTranslation;
  import org.epcglobalinc.tdt.Field;
  import org.epcglobalinc.tdt.GEPC64;
  import org.epcglobalinc.tdt.GEPC64Entry;
  import org.epcglobalinc.tdt.Level;
  import org.epcglobalinc.tdt.LevelTypeList;
  import org.epcglobalinc.tdt.ModeList;
  import org.epcglobalinc.tdt.Option;
  import org.epcglobalinc.tdt.PadDirectionList;
  import org.epcglobalinc.tdt.Rule;
  import org.epcglobalinc.tdt.Scheme;
  import org.w3c.dom.Document;
  import org.xml.sax.SAXException;
  
  /**
   * 
   * This class provides methods for translating an electronic product code (EPC)
   * between various levels of representation including BINARY, TAG_ENCODING,
   * PURE_IDENTITY and LEGACY formats. An additional output level ONS_HOSTNAME may
   * be defined for some coding schemes.
   * 
   * @author Mark Harrison [University of Cambridge] - mark.harrison@cantab.net
   * @author James Brusey
   * @author Jochen Mader - jochen@pramari.com
   * @author Christian Floerkemeier
   */
  public class TDTEngine {
  
  	// --------------------------/
  	// - Class/Member Variables -/
  	// --------------------------/
  
  	/**
  	 * prefix_tree_map is a map of levels to prefix trees. Each prefix tree is a
  	 * Trie structure (see wikipedia) that is useful for quickly finding a
  	 * matching prefix.
  	 */
  	private Map<LevelTypeList, PrefixTree<PrefixMatch>> prefix_tree_map = new HashMap<LevelTypeList, PrefixTree<PrefixMatch>>();
  
  	/**
  	 * HashMap gs1cpi is an associative array providing a lookup between either
  	 * a GS1 Company Prefix and the corresponding integer-based Company Prefix
  	 * Index, where one of these has been registered for use with 64-bit EPCs -
  	 * or the reverse lookup from Company Prefix Index to GS1 Company Prefix.
 	 * Note that this is an optimization to avoid having to do an xpath trawl
 	 * through the CPI table each time.
 	 */
 	private HashMap<String, String> gs1cpi = new HashMap<String, String>();
 
 	/** The gepc64 table xml. */
 	private String GEPC64xml;
 
 	// ----------------/
 	// - Constructors -/
 	// ----------------/
 
 	/**
 	 * Legacy constructor for a new Tag Data Translation engine.
 	 * 
 	 * @param confdir
 	 *            the string value of the path to a configuration directory
 	 *            consisting of two subdirectories, <code>schemes</code> and
 	 *            <code>auxiliary</code>.
 	 * 
 	 *            <p>
 	 *            When the class TDTEngine is constructed, the path to a local
 	 *            directory must be specified, by passing it as a single string
 	 *            parameter to the constructor method, e.g.
 	 *            <code><pre>TDTEngine engine = new TDTEngine("/opt/TDT");</pre></code>
 	 *            </p>
 	 *            <p>
 	 *            The specified directory must contain two subdirectories named
 	 *            auxiliary and schemes. The Tag Data Translation definition
 	 *            files for the various coding schemes should be located inside
 	 *            the subdirectory called <code>schemes</code>. Any auxiliary
 	 *            lookup files (such as <code>ManagerTranslation.xml</code>)
 	 *            should be located inside the subdirectory called
 	 *            <code>auxiliary</code>.
 	 * 
 	 *            Files within the schemes directory ending in <code>.xml</code>
 	 *            are read in and unmarshalled using JAXB.
 	 */
 	@Deprecated
 	public TDTEngine(String confdir) throws FileNotFoundException,
 			MarshalException, ValidationException, TDTException {
 
 		
 		try {
 			Unmarshaller unmar = getUnmarshaller();
 			URL confdirurl;
 			if (confdir.endsWith("/")) {
 				confdirurl = new URL("file","localhost",confdir);
 			} else {
 				confdirurl = new URL("file","localhost",confdir+"/");
 			}
 
 			URL scheme = new URL(confdirurl,"schemes/");
 			URL auxGEPC64table = new URL(confdirurl,"auxiliary/ManagerTranslation.xml");
 
 			Set<String> schemes = new HashSet<String>();
 
 			URLConnection urlcon = scheme.openConnection();
 			urlcon.connect();
 			BufferedReader in = new BufferedReader(new InputStreamReader(urlcon.getInputStream()));
 			String line;
 			for (; (line = in.readLine()) != null;) {
 				if (line.endsWith(".xml")) {
 					URL defurl = new URL(scheme,line);
 					loadEpcTagDataTranslation(unmar, defurl);
 					schemes.add(line);
 				}
 			}
 			loadGEPC64Table(unmar, auxGEPC64table);
 		} catch (MalformedURLException e) {
 			throw new FileNotFoundException(e.getMessage());
 		} catch (IOException e) {
 			throw new FileNotFoundException(e.getMessage());
 		} catch (JAXBException e) {
 			throw new MarshalException(e);
 		}
 
 	}
 
 	/**
 	 * Constructor for a new Tag Data Translation engine. This constructor uses
 	 * the schemes included on the classpath in a directory called schemes 
            * (or from within the jar). The ManagerTranslation.xml file is loaded from a directory
 	* called auxiliary on the classpath. All schemes used need to be listed in a file schemes/schemes.list	   
 	 * 
 	 * @throws IOException
 	 *             thrown if the url is unreachable
 	 * @throws JAXBException
 	 *             thrown if the schemes could not be parsed
 	 */
 	public TDTEngine() throws IOException, JAXBException {
 		
 		Unmarshaller unmar = getUnmarshaller();
 
 		URL auxiliary = this.getClass().getClassLoader().getResource(
 				"auxiliary/ManagerTranslation.xml");
 
 		URL schemes = TDTEngine.class.getClassLoader().getResource(
 				"schemes/schemes.list");
 
 		URLConnection urlcon = schemes.openConnection();
 		urlcon.connect();
 		BufferedReader in = new BufferedReader(new InputStreamReader(urlcon
 				.getInputStream()));
 		String line;
 		for (; (line = in.readLine()) != null;) {
 			loadEpcTagDataTranslation(unmar, TDTEngine.class.getClassLoader()
 					.getResource("schemes/" + line));
 		}
 		loadGEPC64Table(unmar, auxiliary);
 	}
 
 	/**
 	 * Constructor for a new Tag Data Translation engine. All files are
 	 * unmarshalled using JAXB.
 	 * 
 	 * @param auxiliarydir
 	 *            URL to the directory containing auxiliary files such as the GEPC64Table, 
 	 *					"ManagerTranslation.xml"
 	 * @param schemesdir
 	 *            URL to the directory containing the schemes, all files ending in xml are
 	 *            read and parsed
 	 * @throws IOException
 	 *             thrown if the url is unreachable
 	 * @throws JAXBException
 	 *             thrown if the files could not be parsed
 	 */
 	public TDTEngine(URL auxiliarydir, URL schemesdir) throws IOException,
 			JAXBException {
 		Unmarshaller unmar = getUnmarshaller();
 
 		if (!(auxiliarydir.toString().endsWith("/"))) {
 			// if a trailing slash was missing, append a trailing slash
 				auxiliarydir = new URL(auxiliarydir.toString()+"/");
 		}
 		
 		if (!(schemesdir.toString().endsWith("/"))) {
 			// if a trailing slash was missing, append a trailing slash
 				schemesdir = new URL(schemesdir.toString()+"/");
 		}
 		
 		Set<String> requiredauxfiles = new HashSet<String>();
 		requiredauxfiles.add("ManagerTranslation.xml");
 
 		Set<String> schemes = new HashSet<String>();
 		
 		
 		Set<URL> schemeURLs = getschemeURLs(schemesdir,schemes);
 		for (URL defurl : schemeURLs) {
 			loadEpcTagDataTranslation(unmar, defurl);
 		}
 		
 		HashMap<String,URL> auxfileURLs = getauxiliaryURLs(auxiliarydir,requiredauxfiles);
 	
 		URL GEPC64table = auxfileURLs.get("ManagerTranslation.xml");
 		loadGEPC64Table(unmar, GEPC64table);
 	}
 
 
 
 	/**
 	 * Constructor for a new Tag Data Translation engine. All files are
 	 * unmarshalled using JAXB.
 	 * 
 	 * @param auxiliarydir
 	 *            URL to the directory containing auxiliary files such as the GEPC64Table, 
 	 *					"ManagerTranslation.xml"
 	 * @param schemeslist
 	 *            set containing several urls pointing to directories containing
 	 *            the schemes. All files ending in xml are read and parsed.
 	 * @param absolute
 	 *            true if the given URLs are absolute
 	 * @throws IOException
 	 *             thrown if the url is unreachable
 	 * @throws JAXBException
 	 *             thrown if the files could not be parsed
 	 */
 	public TDTEngine(URL auxiliarydir, Set<URL> schemeslist, boolean absolute) throws JAXBException, IOException {
 	
 		Unmarshaller unmar = getUnmarshaller();
 		Set<String> schemes = new HashSet<String>();
 		
 		if (!(auxiliarydir.toString().endsWith("/"))) {
 			// if a trailing slash was missing, append a trailing slash
 				auxiliarydir = new URL(auxiliarydir.toString()+"/");
 		}
 		
 		if (absolute) {
 			for (URL scheme: schemeslist) {
 				loadEpcTagDataTranslation(unmar, scheme);
 			}
 		
 		
 		} else {
 			for (URL schemedir: schemeslist) {
 
 				// if the set<URL> schemes are not absolute URLs, then need to check for trailing slashes
 				if (!(schemedir.toString().endsWith("/"))) {
 					// if a trailing slash was missing, append a trailing slash
 						schemedir = new URL(schemedir.toString()+"/");
 				}
 								
 				Set<URL> schemeURLs = getschemeURLs(schemedir,schemes);
 				for (URL defurl : schemeURLs) {
 					loadEpcTagDataTranslation(unmar, defurl);
 				}
 			}
 		}
 
 
 		URL GEPC64table = new URL(auxiliarydir, "ManagerTranslation.xml");
 		loadGEPC64Table(unmar, GEPC64table);
 	}
 
 	/**
 	 * Creates the unmarshaller.
 	 * 
 	 * @return
 	 * @throws JAXBException
 	 */
 	private Unmarshaller getUnmarshaller() throws JAXBException {
 		JAXBContext context = JAXBContext.newInstance(
 				EpcTagDataTranslation.class, GEPC64.class, GEPC64Entry.class);
 		return context.createUnmarshaller();
 	}
 
 	/**
 	 * Load an xml file from the given url and unmarshal it into an
 	 * EPCTagDataTranslation.
 	 * 
 	 * @param unmar
 	 * @param schemeUrl
 	 * @throws IOException
 	 * @throws JAXBException
 	 */
 	private void loadEpcTagDataTranslation(Unmarshaller unmar, URL schemeUrl)
 			throws IOException, JAXBException {
 		URLConnection urlcon = schemeUrl.openConnection();
 		urlcon.connect();
 		// xml doesn't have enough info for jaxb to figure out
 		// the
 		// classname, so we are doing explicit loading
 		JAXBElement<EpcTagDataTranslation> el = unmar.unmarshal(
 				new StreamSource(urlcon.getInputStream()),
 				EpcTagDataTranslation.class);
 		EpcTagDataTranslation tdt = el.getValue();
 		initFromTDT(tdt);
 	}
 
 	/**
 	 * Load an xml file from the given url and unmarshal it into a GEPC64Table.
 	 * 
 	 * @param unmar
 	 * @param auxiliary
 	 * @throws IOException
 	 * @throws JAXBException
 	 */
 	private void loadGEPC64Table(Unmarshaller unmar, URL auxiliary)
 			throws IOException, JAXBException {
 		URLConnection urlcon = auxiliary.openConnection();
 		urlcon.connect();
 		// load the GEPC64Table
 		JAXBElement<GEPC64> el = unmar.unmarshal(new StreamSource(urlcon
 				.getInputStream()), GEPC64.class);
 		GEPC64 cpilookup = el.getValue();
 		for (GEPC64Entry entry : cpilookup.getEntry()) {
 			String comp = entry.getCompanyPrefix();
 			String indx = entry.getIndex().toString();
 			gs1cpi.put(indx, comp);
 			gs1cpi.put(comp, indx);
 		}
 	}
 	
 		/**
 	 * Private method for obtaining a hashmap of URLs for named auxiliary files, whether in a file directory, web directory or web page
 	 * 
 	 * @param auxdirURL
 	 *            URL to the directory containing auxiliary files or web page linking to auxiliary files
 	 * @param requiredauxfiles
 	 *            Set of individual auxiliary files to be retrieved 
 	 * @return a hash map relating the name of the filename and its URL
 	 *
 	 * @throws IOException
 	 *             thrown if the url is unreachable
 	 */
     private static HashMap<String,URL> getauxiliaryURLs(URL auxdirURL, Set<String> requiredauxfiles) {
 		
 		
 		HashMap<String, URL> foundauxfiles = new HashMap<String, URL>();
 		
 		String inputAuxLine;
 		
 		try {
  			URL parent = new URL(auxdirURL,".");
 						
 			String relauxiliary=auxdirURL.getFile();
 						
             URLConnection urlconauxiliary = auxdirURL.openConnection();
             BufferedReader dis2 = new BufferedReader(new InputStreamReader(urlconauxiliary.getInputStream()));
 			
             while ((inputAuxLine = dis2.readLine()) != null) {
 				
 				for (String requestedfile: requiredauxfiles) {
 					if (requestedfile.endsWith(".xml")) {
 						String pattern = requestedfile.replaceAll("\\.","\\\\.").replaceAll("^","(").replaceAll("$",")").toString();
 						Pattern regex = Pattern.compile(pattern);
 						
 						
 						String pattern2 = requestedfile.replaceAll("\\.","\\\\.").replaceAll("^","href=['\"]([^ ]+?").replaceAll("$",")['\"]").toString();
 						Pattern regex2 = Pattern.compile(pattern2);
 						
 						// check if line includes filename.xml - if so, extract auxiliaryfile           
 						Matcher matcher2 = regex.matcher(inputAuxLine);
 						if (matcher2.find()) {
 							URL relURL = new URL(parent, matcher2.group(1));
 							foundauxfiles.put(requestedfile, relURL);
 						}
 						
 						// check if line includes href="filename.xml - if so, extract auxiliaryfile           
 						Matcher matcher3 = regex2.matcher(inputAuxLine);
 						if (matcher3.find()) {
 							URL relURL = new URL(parent, matcher3.group(1));
 							foundauxfiles.put(requestedfile, relURL);
 						}
 					}
 				}	
 			}
 			
 			dis2.close();
 			
 			
         } catch (MalformedURLException me) {
             System.out.println("MalformedURLException: " + me);
         } catch (IOException ioe) {
             System.out.println("IOException: " + ioe);
         }
 		return foundauxfiles;    
 		
     }
     
 	/**
 	 * Private method for obtaining a set of URLs for TDT definition files, whether in a file directory, web directory or web page
 	 * 
 	 * @param schemesdirURL
 	 *            URL to the directory containing TDT definition files files or web page linking to TDT definition files
 	 * @return a list of URLs of TDT definition files contained within the directory or web page pointed to by the URL
 	 *
 	 * @throws IOException
 	 *             thrown if the url is unreachable
 	 */
     private static Set<URL> getschemeURLs(URL schemesdirURL, Set<String> schemes) {
         String inputSchemeLine;
 
 		Set<URL> schemeURLs = new HashSet<URL>();
 		try {
  			URL parent = new URL(schemesdirURL,".");
 
 			String relschemes=schemesdirURL.getFile();
 
 
 			
             URLConnection urlconschemes = schemesdirURL.openConnection();
             BufferedReader dis = new BufferedReader(new InputStreamReader(urlconschemes.getInputStream()));
 			
             while ((inputSchemeLine = dis.readLine()) != null) {
 				// check if line includes filename.xml - if so, add to Set            
 				Matcher matcher = Pattern.compile("([A-Za-z0-9-_]+-[0-9*]+\\.xml)").matcher(inputSchemeLine);
 				if (matcher.find()) {
 					URL relURL = new URL(parent, matcher.group(0));
 					if (!matcher.group(0).contains("test")) {
 					schemeURLs.add(relURL);
 					schemes.add(matcher.group(0));
 					}
 
  				}
 			}
             dis.close();
         } catch (MalformedURLException me) {
             System.out.println("MalformedURLException: " + me);
         } catch (IOException ioe) {
             System.out.println("IOException: " + ioe);
         }
 		
 		return schemeURLs;
     }
 
 
 
 
 	// -----------/
 	// - Methods -/
 	// -----------/
 
 	private class PrefixMatch {
 		private Scheme s;
 		private Level level;
 
 		public PrefixMatch(Scheme s, Level level) {
 			this.s = s;
 			this.level = level;
 		}
 
 		public Scheme getScheme() {
 			return s;
 		}
 
 		public Level getLevel() {
 			return level;
 		}
 	}
 
 	/** initialise various indices */
 	private void initFromTDT(EpcTagDataTranslation tdt) {
 		for (Scheme ss : tdt.getScheme()) {
 			// create an index so that we can find a scheme based on tag length
 
 			for (Level level : ss.getLevel()) {
 				String s = level.getPrefixMatch();
 				if (s != null) {
 					// insert into prefix tree according to level type.
 					PrefixTree<PrefixMatch> prefix_tree = prefix_tree_map
 							.get(level.getType());
 					if (prefix_tree == null) {
 						prefix_tree = new PrefixTree<PrefixMatch>();
 						prefix_tree_map.put(level.getType(), prefix_tree);
 					}
 					prefix_tree.insert(s, new PrefixMatch(ss, level));
 				}
 			}
 
 		}
 	}
 
 	/**
 	 * Given an input string, and optionally a tag length, find a scheme / level
 	 * with a matching prefix and tag length.
 	 */
 	private PrefixMatch findPrefixMatch(String input, String tagLength) {
 		List<PrefixMatch> match_list = new ArrayList<PrefixMatch>();
 
 		for (PrefixTree<PrefixMatch> tree : prefix_tree_map.values()) {
 
 			List<PrefixMatch> list = tree.search(input);
 
 			if (!list.isEmpty()) {
 				if (tagLength == null)
 					match_list.addAll(list);
 				else {
 
 					for (PrefixMatch match : list) {
 						if (match.getScheme().getTagLength().equals(tagLength))
 							match_list.add(match);
 					}
 				}
 			}
 		}
 
 		if (match_list.isEmpty())
 			throw new TDTException(
 					"No schemes or levels matched the input value");
 		else if (match_list.size() > 1)
 			throw new TDTException(
 					"More than one scheme/level matched the input value");
 		else
 			return match_list.get(0);
 	}
 
 	/**
 	 * Given an input string, level, and optionally a tag length, find a
 	 * matching prefix.
 	 */
 	private PrefixMatch findPrefixMatch(String input, String tagLength,
 			LevelTypeList level_type) {
 		List<PrefixMatch> match_list = new ArrayList<PrefixMatch>();
 		PrefixTree<PrefixMatch> tree = prefix_tree_map.get(level_type);
 		assert tree != null;
 		List<PrefixMatch> list = tree.search(input);
 		if (!list.isEmpty()) {
 			if (tagLength == null)
 				match_list.addAll(list);
 			else {
 				for (PrefixMatch match : list)
 					if (match.getScheme().getTagLength() == tagLength)
 						match_list.add(match);
 			}
 		}
 		if (match_list.isEmpty())
 			throw new TDTException(
 					"No schemes or levels matched the input value");
 		else if (match_list.size() > 1)
 			throw new TDTException(
 					"More than one scheme/level matched the input value");
 		else
 			return match_list.get(0);
 	}
 
 	/**
 	 * Translates the input string of a specified input level to a specified
 	 * outbound level of the same coding scheme. For example, the input string
 	 * value may be a tag-encoding URI and the outbound level specified by
 	 * string outboundlevel may be BINARY, in which case the return value is a
 	 * binary representation expressed as a string.
 	 * 
 	 * <p>
 	 * Note that this version of the method requires that the user specify the
 	 * input level, rather than searching for it. However it still automatically
 	 * finds the scheme used.
 	 * </p>
 	 * 
 	 * @param input
 	 *            input tag coding
 	 * @param inputLevel
 	 *            level such as BINARY, or TAG_ENCODING.
 	 * @param tagLength
 	 *            tag length such as VALUE_64 or VALUE_96.
 	 * @param inputParameters
 	 *            a map with any additional properties.
 	 * @param outputLevel
 	 *            required output level.
 	 * @return output tag coding
 	 */
 	public String convert(String input, LevelTypeList inputLevel,
 			String tagLength, Map<String, String> inputParameters,
 			LevelTypeList outputLevel) {
 
 		PrefixMatch match = findPrefixMatch(input, tagLength, inputLevel);
 
 		return convertLevel(match.getScheme(), match.getLevel(), input,
 				inputParameters, outputLevel);
 	}
 
 	/**
 	 * The convert method translates a String input to a specified outbound
 	 * level of the same coding scheme. For example, the input string value may
 	 * be a tag-encoding URI and the outbound level specified by string
 	 * outboundlevel may be BINARY, in which case the return value is a binary
 	 * representation expressed as a string.
 	 * 
 	 * @param input
 	 *            the identifier to be converted.
 	 * @param inputParameters
 	 *            additional parameters which need to be provided because they
 	 *            cannot always be determined from the input value alone.
 	 *            Examples include the taglength, companyprefixlength and filter
 	 *            values.
 	 * @param outputLevel
 	 *            the outbound level required for the ouput. Permitted values
 	 *            include BINARY, TAG_ENCODING, PURE_IDENTITY, LEGACY and
 	 *            ONS_HOSTNAME.
 	 * @return the identifier converted to the output level.
 	 */
 	public String convert(String input, Map<String, String> inputParameters,
 			LevelTypeList outputLevel) {
 
 		String tagLength = null;
 		if (inputParameters.containsKey("taglength")) {
 			// in principle, the user should provide a
 			// TagLengthList object in the parameter list.
 			String s = inputParameters.get("taglength");
 			tagLength = s;
 		}
 
 		PrefixMatch match = findPrefixMatch(input, tagLength);
 
 		return convertLevel(match.getScheme(), match.getLevel(), input,
 				inputParameters, outputLevel);
 	}
 
 	/**
 	 * convert from a particular scheme / level
 	 */
 	private String convertLevel(Scheme tdtscheme, Level tdtlevel, String input,
 			Map<String, String> inputParameters, LevelTypeList outboundlevel) {
 
 		String outboundstring;
 		Map<String, String> extraparams =
 		// new NoisyMap
 		(new HashMap<String, String>(inputParameters));
 
 		// get the scheme's option key, which is the name of a
 		// parameter whose value is matched to the option key of the
 		// level.
 
 		String optionkey = tdtscheme.getOptionKey();
 		String optionValue = extraparams.get(optionkey);
 		// the name of a parameter which allows the appropriate option
 		// to be selected
 
 		// now consider the various options within the scheme and
 		// level for each option element inside the level, check
 		// whether the pattern attribute matches as a regular
 		// expression
 
 		String matchingOptionKey = null;
 		Option matchingOption = null;
 		Matcher prefixMatcher = null;
 		for (Option opt : tdtlevel.getOption()) {
 			if (optionValue == null || optionValue.equals(opt.getOptionKey())) {
 				// possible match
 
 				Matcher matcher = Pattern.compile(opt.getPattern()).matcher(
 						input);
 				if (matcher.matches()) {
 					if (prefixMatcher != null)
 						throw new TDTException("Multiple patterns matched");
 					prefixMatcher = matcher;
 					matchingOptionKey = opt.getOptionKey();
 					matchingOption = opt;
 				}
 			}
 		}
 		if (prefixMatcher == null)
 			throw new TDTException("No patterns matched");
 
 		optionValue = matchingOptionKey;
 
 		for (Field field : matchingOption.getField()) {
 			BigInteger seq = field.getSeq();
 			String strfieldname = field.getName();
 			String strfieldvalue = prefixMatcher.group(seq.intValue());
 			// System.out.println("   processing field " + strfieldname + " = '"
 			// + strfieldvalue + "'");
 
 			if (field.getCompaction() == null) {
 				// if compaction is null, treat field as an integer
 
 				if (field.getCharacterSet() != null) { // if the character set
 					// is specified
 					Matcher charsetmatcher = Pattern.compile(
 							"^" + field.getCharacterSet() + "$").matcher(
 							strfieldvalue);
 					if (!charsetmatcher.matches()) {
 						throw new TDTException(
 								"field "
 										+ strfieldname
 										+ " ("
 										+ strfieldvalue
 										+ ") does not conform to the allowed character set ("
 										+ field.getCharacterSet() + ") ");
 					}
 				}
 
 				BigInteger bigvalue = null;
 
 				if (tdtlevel.getType() == LevelTypeList.BINARY) { // if the
 					// input was
 					// BINARY
 					bigvalue = new BigInteger(strfieldvalue, 2);
 					extraparams.put(strfieldname, bigvalue.toString());
 				} else {
 					if (field.getDecimalMinimum() != null
 							|| field.getDecimalMaximum() != null)
 						bigvalue = new BigInteger(strfieldvalue);
 					extraparams.put(strfieldname, strfieldvalue);
 				}
 
 				if (field.getDecimalMinimum() != null) { // if the decimal
 					// minimum is
 					// specified
 					BigInteger bigmin = new BigInteger(field
 							.getDecimalMinimum());
 
 					if (bigvalue.compareTo(bigmin) == -1) { // throw an
 						// exception if the
 						// field value is
 						// less than the
 						// decimal minimum
 						throw new TDTException("field " + strfieldname + " ("
 								+ bigvalue + ") is less than DecimalMinimum ("
 								+ field.getDecimalMinimum() + ") allowed");
 					}
 				}
 
 				if (field.getDecimalMaximum() != null) { // if the decimal
 					// maximum is
 					// specified
 					BigInteger bigmax = new BigInteger(field
 							.getDecimalMaximum());
 
 					if (bigvalue.compareTo(bigmax) == 1) { // throw an excpetion
 						// if the field
 						// value is greater
 						// than the decimal
 						// maximum
 						throw new TDTException("field " + strfieldname + " ("
 								+ bigvalue
 								+ ") is greater than DecimalMaximum ("
 								+ field.getDecimalMaximum() + ") allowed");
 					}
 				}
 
 				// after extracting the field, it may be necessary to pad it.
 				padField(extraparams, field);
 
 			} else {
 				// compaction is specified - interpret binary as a string value
 				// using a truncated byte per character
 
 				String compaction = field.getCompaction();
 				PadDirectionList padDir = field.getPadDir();
 				String padchar = field.getPadChar();
 				String s;
 				if ("5-bit".equals(compaction))
 					// "5-bit"
 					s = bin2uppercasefive(strfieldvalue);
 				else if ("6-bit".equals(compaction))
 					// 6-bit
 					s = bin2alphanumsix(strfieldvalue);
 				else if ("7-bit".equals(compaction))
 					// 7-bit
 					s = bin2asciiseven(strfieldvalue);
 				else if ("8-bit".equals(compaction))
 					// 8-bit
 					s = bin2bytestring(strfieldvalue);
 				else
 					throw new Error("unsupported compaction method "
 							+ compaction);
 				extraparams.put(strfieldname, stripPadChar(s, padDir, padchar));
 
 			}
 
 		} // for each field;
 
 		/**
 		 * the EXTRACT rules are performed after parsing the input, in order to
 		 * determine additional fields that are to be derived from the fields
 		 * obtained by the pattern match process
 		 */
 
 		int seq = 0;
 		for (Rule tdtrule : tdtlevel.getRule()) {
 			if (tdtrule.getType() == ModeList.EXTRACT) {
 				assert seq < tdtrule.getSeq().intValue() : "Rule out of sequence order";
 				seq = tdtrule.getSeq().intValue();
 				processRules(extraparams, tdtrule);
 			}
 		}
 
 		/**
 		 * Now we need to consider the corresponding output level and output
 		 * option. The scheme must remain the same, as must the value of
 		 * optionkey (to select the corresponding option element nested within
 		 * the required outbound level)
 		 */
 
 		Level tdtoutlevel = findLevel(tdtscheme, outboundlevel);
 		Option tdtoutoption = findOption(tdtoutlevel, optionValue);
 
 		/**
 		 * the FORMAT rules are performed before formatting the output, in order
 		 * to determine additional fields that are required for preparation of
 		 * the outbound format
 		 */
 
 		seq = 0;
 		for (Rule tdtrule : tdtoutlevel.getRule()) {
 			if (tdtrule.getType() == ModeList.FORMAT) {
 				assert seq < tdtrule.getSeq().intValue() : "Rule out of sequence order";
 				seq = tdtrule.getSeq().intValue();
 				processRules(extraparams, tdtrule);
 			}
 		}
 
 		/**
 		 * Now we need to ensure that all fields required for the outbound
 		 * grammar are suitably padded etc. processPadding takes care of firstly
 		 * padding the non-binary fields if padChar and padDir, length are
 		 * specified then (if necessary) converting to binary and padding the
 		 * binary representation to the left with zeros if the bit string is has
 		 * fewer bits than the bitLength attribute specifies. N.B. TDTv1.1 will
 		 * be more specific about bit-level padding rather than assuming that it
 		 * is always to the left with the zero bit.
 		 */
 
 		// System.out.println(" prior to processPadding, " + extraparams);
 		for (Field field : tdtoutoption.getField()) {
 			// processPadding(extraparams, field, outboundlevel, tdtoutoption);
 
 			padField(extraparams, field);
 			if (outboundlevel == LevelTypeList.BINARY)
 				binaryPadding(extraparams, field);
 
 		}
 
 		/**
 		 * Construct the output from the specified grammar (in ABNF format)
 		 * together with the field values stored in inputparams
 		 */
 
 		outboundstring = buildGrammar(tdtoutoption.getGrammar(), extraparams);
 
 		// System.out.println("final extraparams = " + extraparams);
 		// System.out.println("returned " + outboundstring);
 		return outboundstring;
 	}
 
 	/**
 	 * 
 	 * Converts a binary string into a large integer (numeric string)
 	 */
 	public String bin2dec(String binary) {
 		BigInteger dec = new BigInteger(binary, 2);
 		return dec.toString();
 	}
 
 	/**
 	 * 
 	 * Converts a large integer (numeric string) to a binary string
 	 */
 	public String dec2bin(String decimal) {
 		// TODO: required?
 		if (decimal == null) {
 			decimal = "1";
 		}
 		BigInteger bin = new BigInteger(decimal);
 		return bin.toString(2);
 	}
 
 	/**
 	 * 
 	 * Converts a hexadecimal string to a binary string
 	 */
 	public String hex2bin(String hex) {
 		BigInteger bin = new BigInteger(hex.toLowerCase(), 16);
 		return bin.toString(2);
 	}
 
 	/**
 	 * 
 	 * Converts a binary string to a hexadecimal string
 	 */
 	public String bin2hex(String binary) {
 		BigInteger hex = new BigInteger(binary, 2);
 		return hex.toString(16).toUpperCase();
 	}
 
 	/**
 	 * Returns a string built using a particular grammar. Single-quotes strings
 	 * are counted as literal strings, whereas all other strings appearing in
 	 * the grammar require substitution with the corresponding value from the
 	 * extraparams hashmap.
 	 */
 	private String buildGrammar(String grammar, Map<String, String> extraparams) {
 		StringBuilder outboundstring = new StringBuilder();
 		String[] fields = Pattern.compile("\\s+").split(grammar);
 		for (int i = 0; i < fields.length; i++) {
 			if (fields[i].substring(0, 1).equals("'")) {
 				outboundstring.append(fields[i].substring(1,
 						fields[i].length() - 1));
 			} else {
 				outboundstring.append(extraparams.get(fields[i]));
 			}
 		}
 
 		return outboundstring.toString();
 	}
 
 	/**
 	 * 
 	 * Converts the value of a specified fieldname from the extraparams map into
 	 * binary, either handling it as a large integer or taking into account the
 	 * compaction of each ASCII byte that is specified in the TDT definition
 	 * file for that particular field
 	 */
 	private String fieldToBinary(Field field, Map<String, String> extraparams) {
 		// really need an index to find field number given fieldname;
 
 		String fieldname = field.getName();
 		String value = extraparams.get(fieldname);
 		String compaction = field.getCompaction();
 
 		if (compaction == null) {
 			value = dec2bin(value);
 		} else {
 			if ("5-bit".equals(compaction)) {
 				value = uppercasefive2bin(value);
 			} else if ("6-bit".equals(compaction)) {
 				value = alphanumsix2bin(value);
 			} else if ("7-bit".equals(compaction)) {
 				value = asciiseven2bin(value);
 			} else if ("8-bit".equals(compaction)) {
 				value = bytestring2bin(value);
 			} else
 				throw new Error("Unsupported compaction " + compaction);
 		}
 
 		return value;
 	}
 
 	/**
 	 * pad a value according the field definition.
 	 */
 	private void padField(Map<String, String> extraparams, Field field) {
 		String name = field.getName();
 		String value = extraparams.get(name);
 		PadDirectionList padDir = field.getPadDir();
 		int requiredLength = 0;
 		if (field.getLength() != null) {
 			requiredLength = field.getLength().intValue();
 		}
 
 		// assert value != null;
 		if (value == null)
 			return;
 
 		String padCharString = field.getPadChar();
 		// if no pad char specified, don't attempt padding
 		if (padCharString == null)
 			return;
 		assert padCharString.length() > 0;
 		char padChar = padCharString.charAt(0);
 
 		StringBuilder buf = new StringBuilder(requiredLength);
 		if (padDir == PadDirectionList.LEFT) {
 			for (int i = 0; i < requiredLength - value.length(); i++)
 				buf.append(padChar);
 			buf.append(value);
 		} else if (padDir == PadDirectionList.RIGHT) {
 			buf.append(value);
 			for (int i = 0; i < requiredLength - value.length(); i++)
 				buf.append(padChar);
 		}
 		assert buf.length() == requiredLength;
 		if (requiredLength != value.length()) {
 			// System.out.println("    updated " + name + " to '" + buf + "'");
 			extraparams.put(name, buf.toString());
 		}
 		/*
 		 * else { StringBuilder mybuf = new StringBuilder(); for (int i = 0; i <
 		 * value.length(); i++) { if (i > 0) mybuf.append(',');
 		 * mybuf.append('\''); mybuf.append(value.charAt(i));
 		 * mybuf.append('\''); }
 		 * 
 		 * 
 		 * System.out.println("    field " + name + " not padded as " +
 		 * mybuf.toString() + " is already " + requiredLength +
 		 * " characters long"); }
 		 */
 	}
 
 	/**
 	 * If the outbound level is BINARY, convert the string field to binary, then
 	 * pad to the left with the appropriate number of zero bits to reach a
 	 * number of bits specified by the bitLength attribute of the TDT definition
 	 * file.
 	 */
 
 	private void binaryPadding(Map<String, String> extraparams, Field tdtfield) {
 		String fieldname = tdtfield.getName();
 		int reqbitlength = tdtfield.getBitLength().intValue();
 		String value;
 
 		String binaryValue = fieldToBinary(tdtfield, extraparams);
 		if (binaryValue.length() < reqbitlength) {
 			int extraBitLength = reqbitlength - binaryValue.length();
 
 			StringBuilder zeroPaddedBinaryValue = new StringBuilder("");
 			for (int i = 0; i < extraBitLength; i++) {
 				zeroPaddedBinaryValue.append("0");
 			}
 			zeroPaddedBinaryValue.append(binaryValue);
 			value = zeroPaddedBinaryValue.toString();
 		} else {
 			if (binaryValue.length() > reqbitlength)
 				throw new TDTException("Binary value [" + binaryValue
 						+ "] for field " + fieldname
 						+ " exceeds maximum allowed " + reqbitlength
 						+ " bits.  Decimal value was "
 						+ extraparams.get(fieldname));
 
 			value = binaryValue;
 		}
 		extraparams.put(fieldname, value);
 
 	}
 
 	/**
 	 * Removes leading or trailing characters equal to padchar from the
 	 * start/end of the string specified as the first parameter. The second
 	 * parameter specified the stripping direction as "LEFT" or "RIGHT" and the
 	 * third parameter specifies the character to be stripped.
 	 */
 	private String stripPadChar(String field, PadDirectionList paddir,
 			String padchar) {
 		String rv;
 		if (paddir == null || padchar == null)
 			rv = field;
 		else {
 			String pattern;
 			if (paddir == PadDirectionList.RIGHT)
 				pattern = "[" + padchar + "]+$";
 			else
 				// if (paddir == PadDirectionList.LEFT)
 				pattern = "^[" + padchar + "]+";
 
 			rv = field.replaceAll(pattern, "");
 
 		}
 		return rv;
 	}
 
 	/**
 	 * 
 	 * Adds additional entries to the extraparams hashmap by processing various
 	 * rules defined in the TDT definition files. Typically used for string
 	 * processing functions, lookup in tables, calculation of check digits etc.
 	 */
 	private void processRules(Map<String, String> extraparams, Rule tdtrule) {
 		String tdtfunction = tdtrule.getFunction();
 		int openbracket = tdtfunction.indexOf("(");
 		assert openbracket != -1;
 		String params = tdtfunction.substring(openbracket + 1, tdtfunction
 				.length() - 1);
 		String rulename = tdtfunction.substring(0, openbracket);
 		String[] parameter = params.split(",");
 		String newfieldname = tdtrule.getNewFieldName();
 		// System.out.println(tdtfunction + " " + parameter[0] + " " +
 		// extraparams.get(parameter[0]));
 		/**
 		 * Stores in the hashmap extraparams the value obtained from a lookup in
 		 * a specified XML table.
 		 * 
 		 * The first parameter is the given value already known. This is denoted
 		 * as $1 in the corresponding XPath expression
 		 * 
 		 * The second parameter is the string filename of the table which must
 		 * be present in the auxiliary subdirectory
 		 * 
 		 * The third parameter is the column in which the supplied input value
 		 * should be sought
 		 * 
 		 * The fourth parameter is the column whose value should be read for the
 		 * corresponding row, in order to obtain the result of the lookup.
 		 * 
 		 * The rule in the definition file may contain an XPath expression and a
 		 * URL where the table may be obtained.
 		 */
 		if (rulename.equals("TABLELOOKUP")) {
 			// parameter[0] is given value
 			// parameter[1] is table
 			// parameter[2] is input column supplied
 			// parameter[3] is output column required
 			assert parameter.length == 4 : "incorrect number of parameters to tablelookup "
 					+ params;
 			if (parameter[1].equals("tdt64bitcpi")) {
 				String s = extraparams.get(parameter[0]);
 				assert s != null : tdtfunction + " when " + parameter[0]
 						+ " is null";
 				String t = gs1cpi.get(s);
 				assert t != null : "gs1cpi[" + s + "] is null";
 				assert newfieldname != null;
 				extraparams.put(newfieldname, t);
 				// extraparams.put(newfieldname,
 				// gs1cpi.get(extraparams.get(parameter[0])));
 			} else { // JPB! the following is untested
 				String tdtxpath = tdtrule.getTableXPath();
 				String tdttableurl = tdtrule.getTableURL();
 				String tdtxpathsub = tdtxpath.replaceAll("\\$1", extraparams
 						.get(parameter[0]));
 				extraparams.put(newfieldname, xpathlookup(
 						"ManagerTranslation.xml", tdtxpathsub));
 			}
 		}
 
 		/**
 		 * Stores the length of the specified string under the new fieldname
 		 * specified by the corresponding rule of the definition file.
 		 */
 		if (rulename.equals("LENGTH")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction
 					+ " when " + parameter[0] + " is null";
 			if (extraparams.get(parameter[0]) != null) {
 				extraparams.put(newfieldname, Integer.toString(extraparams.get(
 						parameter[0]).length()));
 			}
 		}
 
 		/**
 		 * Stores a GS1 check digit in the extraparams hashmap, keyed under the
 		 * new fieldname specified by the corresponding rule of the definition
 		 * file.
 		 */
 		if (rulename.equals("GS1CHECKSUM")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction
 					+ " when " + parameter[0] + " is null";
 			if (extraparams.get(parameter[0]) != null) {
 				extraparams.put(newfieldname, gs1checksum(extraparams
 						.get(parameter[0])));
 			}
 		}
 
 		/**
 		 * Obtains a substring of the string provided as the first parameter. If
 		 * only a single second parameter is specified, then this is considered
 		 * as the start index and all characters from the start index onwards
 		 * are stored in the extraparams hashmap under the key named
 		 * 'newfieldname' in the corresponding rule of the definition file. If a
 		 * second and third parameter are specified, then the second parameter
 		 * is the start index and the third is the length of characters
 		 * required. A substring consisting characters from the start index up
 		 * to the required length of characters is stored in the extraparams
 		 * hashmap, keyed under the new fieldname specified by the corresponding
 		 * rule of the defintion file.
 		 */
 		if (rulename.equals("SUBSTR")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction
 					+ " when " + parameter[0] + " is null";
 			if (parameter.length == 2) {
 				if (extraparams.get(parameter[0]) != null) {
 					int start = getIntValue(parameter[1], extraparams);
 					if (start >= 0) {
 						extraparams.put(newfieldname, extraparams.get(
 								parameter[0]).substring(start));
 					}
 				}
 
 			}
 			if (parameter.length == 3) { // need to check that this variation is
 				// correct - c.f. Perl substr
 				assert extraparams.get(parameter[0]) != null : tdtfunction
 						+ " when " + parameter[0] + " is null";
 				if (extraparams.get(parameter[0]) != null) {
 					int start = getIntValue(parameter[1], extraparams);
 					int end = getIntValue(parameter[2], extraparams);
 					if ((start >= 0) && (end >= 0)) {
 						extraparams.put(newfieldname, extraparams.get(
 								parameter[0]).substring(start, start + end));
 					}
 				}
 
 			}
 		}
 
 		/**
 		 * Concatenates specified string parameters together. Literal values
 		 * must be enclosed within single or double quotes or consist of
 		 * unquoted digits. Other unquoted strings are considered as fieldnames
 		 * and the corresponding value from the extraparams hashmap are
 		 * inserted. The result of the concatenation (and substitution) of the
 		 * strings is stored as a new entry in the extraparams hashmap, keyed
 		 * under the new fieldname specified by the rule.
 		 */
 		if (rulename.equals("CONCAT")) {
 			StringBuilder buffer = new StringBuilder();
 			for (int p1 = 0; p1 < parameter.length; p1++) {
 				Matcher matcher = Pattern.compile("\"(.*?)\"|'(.*?)'|[0-9]")
 						.matcher(parameter[p1]);
 				if (matcher.matches()) {
 					buffer.append(parameter[p1]);
 				} else {
 					assert extraparams.get(parameter[p1]) != null : tdtfunction
 							+ " when " + parameter[p1] + " is null";
 					if (extraparams.get(parameter[p1]) != null) {
 						buffer.append(extraparams.get(parameter[p1]));
 					}
 				}
 
 			}
 			extraparams.put(newfieldname, buffer.toString());
 		}
 		
 		
 
 		/**
 		 * Adds specified parameters together. Unqouted strings are considered
 		 * as fieldnames and the corresponding value from the extraparams hashmap
 		 * are used in the calculation. 
 		 * The result of the addition is stored as a new entry in the extraparams
 		 * hashmap, keyed under the new fieldname specified by the rule.
 		 */
 		if (rulename.equalsIgnoreCase("add")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction + " when " + parameter[0] + " is null";
 
 			if ((extraparams.get(parameter[0]) != null) && (parameter[1] != null) && (parameter.length == 2)) {
 			
 				int initialvalue = getIntValue(parameter[0], extraparams);
 				int increment = Integer.parseInt(parameter[1]);
 				extraparams.put(newfieldname, Integer.toString(initialvalue+increment));
 			}
 		}
 
 
 
 		/**
 		 * Multiplies the specified parameters together. 
 		 * Unquoted strings are considered as fieldnames and the corresponding
 		 * value from the extraparams hashmap are used in the calculation. 
 		 * The result of the multiplication is stored as a new entry in the 
 		 * extraparams hashmap, keyed under the new fieldname specified by the rule.
 		 */
 		if (rulename.equalsIgnoreCase("multiply")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction + " when " + parameter[0] + " is null";
 
 			if ((extraparams.get(parameter[0]) != null) && (parameter[1] != null) && (parameter.length == 2)) {
 			
 				int initialvalue = getIntValue(parameter[0], extraparams);
 				int factor = Integer.parseInt(parameter[1]);
 				extraparams.put(newfieldname, Integer.toString(initialvalue*factor));
 			}
 		}
 
 
 		/**
 		 * Divides the first parameter by the second parameter. 
 		 * Unquoted strings are considered as fieldnames and the corresponding 
 		 * value from the extraparams hashmap are used in the calculation. 
 		 * The result of the division is stored as a new entry in the 
 		 * extraparams hashmap, keyed under the new fieldname specified by the rule.
 		 */
 		if (rulename.equalsIgnoreCase("divide")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction + " when " + parameter[0] + " is null";
 
 			if ((extraparams.get(parameter[0]) != null) && (parameter[1] != null) && (parameter.length == 2)) {
 			
 				int initialvalue = getIntValue(parameter[0], extraparams);
 				int divisor = Integer.parseInt(parameter[1]);
 				extraparams.put(newfieldname, Integer.toString(initialvalue*divisor));
 			}
 		}
 
 		/**
 		 * Subtracts the second parameter from the first parameter. 
 		 * Unquoted strings are considered as fieldnames and the corresponding 
 		 * value from the extraparams hashmap are used in the calculation. 
 		 * The result of the subtraction is stored as a new entry 
 		 * in the extraparams hashmap, keyed under the new fieldname specified
 		 * by the rule.
 		 */
 		if (rulename.equalsIgnoreCase("subtract")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction + " when " + parameter[0] + " is null";
 
 			if ((extraparams.get(parameter[0]) != null) && (parameter[1] != null) && (parameter.length == 2)) {
 			
 				int initialvalue = getIntValue(parameter[0], extraparams);
 				int decrement = Integer.parseInt(parameter[1]);
 				extraparams.put(newfieldname, Integer.toString(initialvalue-decrement));
 			}
 		}
 
 
 		/**
 		 * Returns the remainder after integer division of the first parameter
 		 * divided by the second parameter. 
 		 * Unquoted strings are considered as fieldnames and the corresponding 
 		 * value from the extraparams hashmap are used in the calculation. 
 		 * The remainder after integer division is stored as a new entry 
 		 * in the extraparams hashmap, keyed under the new fieldname specified
 		 * by the rule.
 		 */
 		if (rulename.equalsIgnoreCase("mod")) {
 			assert extraparams.get(parameter[0]) != null : tdtfunction + " when " + parameter[0] + " is null";
 
 			if ((extraparams.get(parameter[0]) != null) && (parameter[1] != null) && (parameter.length == 2)) {
 			
 				int initialvalue = getIntValue(parameter[0], extraparams);
 				int divisor = Integer.parseInt(parameter[1]);
 				extraparams.put(newfieldname, Integer.toString(initialvalue % divisor));
 			}
 		}
 
 		
 		
 	}
 
 	/**
 	 * 
 	 * Returns the value of a specified fieldname from the specified hashmap and
 	 * returns an integer value or throws an exception if the value is not an
 	 * integer
 	 */
 	private int getIntValue(String fieldname, Map<String, String> extraparams) {
 		Matcher checkint = Pattern.compile("^\\d+$").matcher(fieldname);
 		int rv;
 		if (checkint.matches()) {
 			rv = Integer.parseInt(fieldname);
 		} else {
 			if (extraparams.containsKey(fieldname)) {
 				rv = Integer.parseInt(extraparams.get(fieldname));
 			} else {
 				rv = -1;
 				throw new TDTException("No integer value for " + fieldname
 						+ " can be found - check extraparams");
 			}
 		}
 		return rv;
 	}
 
 	/**
 	 * 
 	 * Performs an XPATH lookup in an xml document. The document has been loaded
 	 * into a private member. The XPATH expression is supplied as the second
 	 * string parameter The return value is of type string e.g. this is
 	 * currently used primarily for looking up the Company Prefix Index for
 	 * encoding a GS1 Company Prefix into a 64-bit EPC tag
 	 * 
 	 */
 	private String xpathlookup(String xml, String expression) {
 
 		try {
 
 			// Parse the XML as a W3C document.
 			DocumentBuilder builder = DocumentBuilderFactory.newInstance()
 					.newDocumentBuilder();
 			Document document = builder.parse(GEPC64xml);
 
 			XPath xpath = XPathFactory.newInstance().newXPath();
 
 			String rv = (String) xpath.evaluate(expression, document,
 					XPathConstants.STRING);
 
 			return rv;
 
 		} catch (ParserConfigurationException e) {
 			System.err.println("ParserConfigurationException caught...");
 			e.printStackTrace();
 			return null;
 		} catch (XPathExpressionException e) {
 			System.err.println("XPathExpressionException caught...");
 			e.printStackTrace();
 			return null;
 		} catch (SAXException e) {
 			System.err.println("SAXException caught...");
 			e.printStackTrace();
 			return null;
 		} catch (IOException e) {
 			System.err.println("IOException caught...");
 			e.printStackTrace();
 			return null;
 		}
 
 	}
 
 	// auxiliary functions
 
 	/**
 	 * 
 	 * Converts a binary string input into a byte string, using 8-bits per
 	 * character byte
 	 */
 	private String bytestring2bin(String bytestring) {
 		String binary;
 		StringBuilder buffer = new StringBuilder("");
 		int len = bytestring.length();
 		byte[] bytes = bytestring.getBytes();
 		for (int i = 0; i < len; i++) {
 			buffer.append(padBinary(dec2bin(Integer.toString(bytes[i])), 8));
 		}
 		binary = buffer.toString();
 		return binary;
 	}
 
 	/**
 	 * 
 	 * Converts a byte string input into a binary string, using 8-bits per
 	 * character byte
 	 */
 	private String bin2bytestring(String binary) {
 		String bytestring;
 		StringBuilder buffer = new StringBuilder("");
 		int len = binary.length();
 		for (int i = 0; i < len; i += 8) {
 			int j = Integer.parseInt(bin2dec(padBinary(binary.substring(i,
 					i + 8), 8)));
 			buffer.append((char) j);
 		}
 		bytestring = buffer.toString();
 		return bytestring;
 	}
 
 	/**
 	 * 
 	 * Converts an ASCII string input into a binary string, using 7-bit
 	 * compaction of each ASCII byte
 	 */
 	private String asciiseven2bin(String asciiseven) {
 		String binary;
 		StringBuilder buffer = new StringBuilder("");
 		int len = asciiseven.length();
 		byte[] bytes = asciiseven.getBytes();
 		for (int i = 0; i < len; i++) {
 			buffer.append(padBinary(dec2bin(Integer.toString(bytes[i] % 128)),
 					8).substring(1, 8));
 		}
 		binary = buffer.toString();
 		return binary;
 	}
 
 	/**
 	 * 
 	 * Converts a binary string input into an ASCII string output, assuming that
 	 * 7-bit compaction was used
 	 */
 	private String bin2asciiseven(String binary) {
 		String asciiseven;
 		StringBuilder buffer = new StringBuilder("");
 		int len = binary.length();
 		for (int i = 0; i < len; i += 7) {
 			int j = Integer.parseInt(bin2dec(padBinary(binary.substring(i,
 					i + 7), 8)));
 			buffer.append((char) j);
 		}
 		asciiseven = buffer.toString();
 		return asciiseven;
 	}
 
 	/**
 	 * Converts an alphanumeric string input into a binary string, using 6-bit
 	 * compaction of each ASCII byte
 	 */
 	private String alphanumsix2bin(String alphanumsix) {
 		String binary;
 		StringBuilder buffer = new StringBuilder("");
 		int len = alphanumsix.length();
 		byte[] bytes = alphanumsix.getBytes();
 		for (int i = 0; i < len; i++) {
 			buffer
 					.append(padBinary(dec2bin(Integer.toString(bytes[i] % 64)),
 							8).substring(2, 8));
 		}
 		binary = buffer.toString();
 		return binary;
 	}
 
 	/**
 	 * 
 	 * Converts a binary string input into a character string output, assuming
 	 * that 6-bit compaction was used
 	 */
 	private String bin2alphanumsix(String binary) {
 		String alphanumsix;
 		StringBuilder buffer = new StringBuilder("");
 		int len = binary.length();
 		for (int i = 0; i < len; i += 6) {
 			int j = Integer.parseInt(bin2dec(padBinary(binary.substring(i,
 					i + 6), 8)));
 			if (j < 32) {
 				j += 64;
 			}
 			buffer.append((char) j);
 		}
 		alphanumsix = buffer.toString();
 		return alphanumsix;
 	}
 
 	/**
 	 * Converts an upper case character string input into a binary string, using
 	 * 5-bit compaction of each ASCII byte
 	 */
 	private String uppercasefive2bin(String uppercasefive) {
 		String binary;
 		StringBuilder buffer = new StringBuilder("");
 		int len = uppercasefive.length();
 		byte[] bytes = uppercasefive.getBytes();
 		for (int i = 0; i < len; i++) {
 			buffer
 					.append(padBinary(dec2bin(Integer.toString(bytes[i] % 32)),
 							8).substring(3, 8));
 		}
 		binary = buffer.toString();
 		return binary;
 	}
 
 	/**
 	 * 
 	 * Converts a binary string input into a character string output, assuming
 	 * that 5-bit compaction was used
 	 */
 	private String bin2uppercasefive(String binary) {
 		String uppercasefive;
 		StringBuilder buffer = new StringBuilder("");
 		int len = binary.length();
 		for (int i = 0; i < len; i += 5) {
 			int j = Integer.parseInt(bin2dec(padBinary(binary.substring(i,
 					i + 5), 8)));
 			buffer.append((char) (j + 64));
 		}
 		uppercasefive = buffer.toString();
 		return uppercasefive;
 	}
 
 	/**
 	 * Pads a binary value supplied as a string first parameter to the left with
 	 * leading zeros in order to reach a required number of bits, as expressed
 	 * by the second parameter, reqlen. Returns a string value corresponding to
 	 * the binary value left padded to the required number of bits.
 	 */
 	private String padBinary(String binary, int reqlen) {
 		String rv;
 		int l = binary.length();
 		int pad = (reqlen - (l % reqlen)) % reqlen;
 		StringBuilder buffer = new StringBuilder("");
 		for (int i = 0; i < pad; i++) {
 			buffer.append("0");
 		}
 		buffer.append(binary);
 		rv = buffer.toString();
 		return rv;
 	}
 
 	/**
 	 * Calculates the check digit for a supplied input string (assuming that the
 	 * check digit will be the digit immediately following the supplied input
 	 * string). GS1 (formerly EAN.UCC) check digit calculation methods are used.
 	 */
 	private String gs1checksum(String input) {
 		int checksum;
 		int weight;
 		int total = 0;
 		int len = input.length();
 		int d;
 		for (int i = 0; i < len; i++) {
 			if (i % 2 == 0) {
 				weight = -3;
 			} else {
 				weight = -1;
 			}
 			d = Integer.parseInt(input.substring(len - 1 - i, len - i));
 			total += weight * d;
 		}
 		checksum = (10 + total % 10) % 10;
 		return Integer.toString(checksum);
 	}
 
 	/**
 	 * find a level by its type in a scheme. This involves iterating through the
 	 * list of levels. The main reason for doing it this way is to avoid being
 	 * dependent on the order in which the levels are coded in the xml, which is
 	 * not explicitly constrained.
 	 */
 	private Level findLevel(Scheme scheme, LevelTypeList levelType) {
 		Level level = null;
 		for (Level lev : scheme.getLevel()) {
 			if (lev.getType() == levelType) {
 				level = lev;
 				break;
 			}
 		}
 		if (level == null)
 			throw new Error("Couldn't find type " + levelType + " in scheme "
 					+ scheme);
 		return level;
 	}
 
 	/**
 	 * find a option by its type in a scheme. This involves iterating through
 	 * the list of options. The main reason for doing it this way is to avoid
 	 * being dependent on the order in which the options are coded in the xml,
 	 * which is not explicitly constrained.
 	 */
 	private Option findOption(Level level, String optionKey) {
 		Option option = null;
 
 		for (Option opt : level.getOption()) {
 			if (opt.getOptionKey().equals(optionKey)) {
 				option = opt;
 				break;
 			}
 		}
 		if (option == null)
 			throw new Error("Couldn't find option for " + optionKey
 					+ " in level " + level);
 
 		return option;
 	}
 	
 	/** 
 	 * adds a list of global company prefixes (GCPs) to the current list of GCPs.
 	 * The list of GCPs is used to convert a GTIN and serial or an SSCC to an 
 	 * EPC number when the user does not provide length of the GCP.
 	 * 
 	 * The method expects the individual GCPs to be on a new line each. It is up 
 	 * to the user to determine wher the GCPs are read from (normal file, network, 
 	 * onsepc.com)
 	 * 
 	 *  @param inputstream 
 	 *  			a reference to a source of GCPs 
 	 * @throws IOException 
 	 */
 	
 	public void addListOfGCPs(InputStream source) throws IOException {
 		
 		BufferedReader br = new BufferedReader(new InputStreamReader(
 	            source, "US-ASCII"));
 	    try {
 	      String line;
 	      while ((line = br.readLine()) != null) {
 	        //System.out.println(line);
 	      }
 	    } finally {
 	      br.close();
 	    }	   
 	}
 	
 	/**
 	 * converts a GTIN and serial number to the pure identity representation of an EPC. 
 	 * The method looks up the length of the global company prefix from a list that can 
 	 * loaded into the TDT engine.
 	 * 
 	 *  @params gtin
 	 *  @params serial
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertGTINandSerialToPureIdentityEPC(String gtin, String serial) {
 		
 		return " ";
 		
 	}
 	
 	/**
 	 * converts a GTIN and serial number to the pure identity representation of an EPC. 
 	 * The length of the global company prefix is provided as a method parameter.
 	 * 
 	 *  @params gtin
 	 *  @params serial
 	 *  @params length of global company prefix
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertGTINandSerialToPureIdentityEPC(String gtin, String serial, int gcpLength) {
 		
 		return " ";
 		
 	}
 	
 	/**
 	 * converts a pure identity EPC to gtin and serial. 
 	 * 
 	 *  @params epc in pure identity format
 	 *  @returns List with gtin and serial
 	 *  
 	 */ 
 	
 	public List<String> convertPureIdentityEPCToGTINandSerial(String EPC) {
 		
 		return new ArrayList<String>();
 		
 	}
 	
 	
 	/**
 	 * converts a SSCC to the pure identity representation of an EPC. The method looks up 
 	 * the length of the global company prefix from a list that can loaded into the TDT 
 	 * engine via the addGCPs  
 	 * 
 	 *  @params gtin
 	 *  @params serial
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertSSCCToPureIdentityEPC(String sscc) {
 		
 		return " ";
 		
 	}
 	
 	/**
 	 * converts a SSCC to the pure identity representation of an EPC. 
 	 * The length of the global company prefix is provided as a method parameter.
 	 *  @params gtin
 	 *  @params serial
 	 *  @params length of global company prefix
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertSSCCToPureIdentityEPC(String sscc, int gcpLength) {
 		
 		return " ";
 		
 	}
 	
 	/**
 	 * converts a pure identity EPC to gtin and serial. 
 	 * 
 	 *  @params epc in pure identity format
 	 *  @returns List with gtin and serial
 	 *  
 	 */ 
 	
 	public String convertPureIdentityEPCToSSCC(String EPC) {
 		
 		return " ";
 		
 	}
 	
 	
 	
 	/**
 	 * converts a GLN and serial to the pure identity representation of an EPC. The method looks up 
 	 * the length of the global company prefix from a list that can loaded into the TDT 
 	 * engine. 
 	 * 
 	 *  @params gtin
 	 *  @params serial
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertGLNandSerialToPureIdentityEPC(String gln, String serial) {
 		
 		return " ";
 		
 	}
 	
 	/**
 	 * converts a GLN and serial to the pure identity representation of an EPC.
 	 * The length of the global company prefix is provided as a method parameter.
 	 *  @params gtin
 	 *  @params serial
 	 *  @params length of global company prefix
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertGLNandSerialToPureIdentityEPC(String gln, String serial, int gcpLength) {
 		
 		return " ";
 		
 	}
 	
 	
 	
 	/**
 	 * converts a binary EPC to a pure identity representation. 
 	 *  @params binary EPC
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertBinaryEPCToPureIdentityEPC(String binary) {
 		
 		return " ";
 		
 	}
 	
 	/**
 	 * converts a binary EPC in hex notation to a pure identity representation. 
 	 *  @params hexadecimal EPC
 	 *  @returns pure identity EPC
 	 *  
 	 */ 
 	
 	public String convertHexEPCToPureIdentityEPC(String binary) {
 		
 		return " ";
 		
 	}
 	
 
 	
 }