package uk.ac.vamsas.client.simpleclient; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.Hashtable; import java.util.Iterator; import java.util.Vector; import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import uk.ac.vamsas.client.AppDataOutputStream; import uk.ac.vamsas.client.ClientHandle; import uk.ac.vamsas.client.IVorbaIdFactory; import uk.ac.vamsas.client.SessionHandle; import uk.ac.vamsas.client.UserHandle; import uk.ac.vamsas.client.Vobject; import uk.ac.vamsas.client.VorbaIdFactory; import uk.ac.vamsas.client.VorbaXmlBinder; import uk.ac.vamsas.objects.core.ApplicationData; import uk.ac.vamsas.objects.core.VAMSAS; import uk.ac.vamsas.objects.core.VamsasDocument; import uk.ac.vamsas.objects.utils.AppDataReference; import uk.ac.vamsas.objects.utils.DocumentStuff; import uk.ac.vamsas.objects.utils.ProvenanceStuff; import uk.ac.vamsas.objects.utils.document.VersionEntries; /** * Class for high-level io and Jar manipulation involved in creating * or updating a vamsas archive (with backups). * Writes to a temporary file and then swaps new file for backup. * uses the sessionFile locking mechanism for safe I/O * @author jimp * */ public class VamsasArchive { private static Log log = LogFactory.getLog(VamsasArchive.class); /** * Access original document if it exists, and get VAMSAS root objects. * @return vector of vamsas roots from original document * @throws IOException */ public static Vobject[] getOriginalRoots(VamsasArchive ths) throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { VamsasArchiveReader oReader = ths.getOriginalArchiveReader(); if (oReader!=null) { if (oReader.isValid()) { InputStreamReader vdoc = new InputStreamReader(oReader.getVamsasDocumentStream()); VamsasDocument doc = VamsasDocument.unmarshal(vdoc); if (doc!=null) return doc.getVAMSAS(); // TODO ensure embedded appDatas are garbage collected to save memory } else { InputStream vxmlis = oReader.getVamsasXmlStream(); if (vxmlis!=null) { // Might be an old vamsas file. BufferedInputStream ixml = new BufferedInputStream(oReader.getVamsasXmlStream()); InputStreamReader vxml = new InputStreamReader(ixml); VAMSAS root[] = new VAMSAS[1]; root[0] = VAMSAS.unmarshal(vxml); if (root[0]!=null) return root; } } } return null; } /** * Access the original vamsas document for a VamsasArchive class, and return it. * Users of the VamsasArchive class should use the getVamsasDocument method to retrieve * the current document - only use this one if you want the 'backup' version. * TODO: catch OutOfMemoryError - they are likely to occur here. * NOTE: vamsas.xml datastreams are constructed as 'ALPHA_VERSION' vamsas documents. * @param ths * @return null if no document exists. * @throws IOException * @throws org.exolab.castor.xml.MarshalException * @throws org.exolab.castor.xml.ValidationException */ public static VamsasDocument getOriginalVamsasDocument(VamsasArchive ths) throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { return VamsasArchive.getOriginalVamsasDocument(ths, null); } /** * Uses VorbaXmlBinder to retrieve the VamsasDocument from the original archive referred to by ths * @param ths * @param vorba * @return * @throws IOException * @throws org.exolab.castor.xml.MarshalException * @throws org.exolab.castor.xml.ValidationException */ public static VamsasDocument getOriginalVamsasDocument(VamsasArchive ths, VorbaIdFactory vorba) throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { VamsasArchiveReader oReader = ths.getOriginalArchiveReader(); if (oReader!=null) { ths.setVorba(vorba); return ths.vorba.getVamsasDocument(oReader); } // otherwise - there was no valid original document to read. return null; } /** * destination of new archive data (tempfile if virginarchive=true, original archive location otherwise) */ java.io.File archive=null; /** * locked IO handler for new archive file */ SessionFile rchive=null; /** * original archive file to be updated (or null if virgin) where new data will finally reside */ java.io.File original=null; /** * original archive IO handler */ SessionFile odoclock = null; Lock destinationLock = null; /** * Original archive reader class */ VamsasArchiveReader odoc = null; /** * true if a real vamsas document is being written. */ boolean vamsasdocument=true; /** * Output stream for archived data */ org.apache.tools.zip.ZipOutputStream newarchive=null; /** * JarEntries written to archive */ Hashtable entries = null; /** * true if we aren't just updating an archive */ private boolean virginArchive=false; /** * name of backup of existing archive that has been updated/overwritten. * only one backup will be made - and this is it. */ File originalBackup = null; boolean donotdeletebackup=false; private final int _TRANSFER_BUFFER=4096*4; protected SimpleDocument vorba = null; /** * LATER: ? CUT'n'Paste error ? * Access and return current vamsas Document, if it exists, or create a new one * (without affecting VamsasArchive object state - so is NOT THREAD SAFE) * _TODO: possibly modify internal state to lock low-level files * (like the IClientDocument interface instance constructer would do) * @see org.vamsas.simpleclient.VamsasArchive.getOriginalVamsasDocument for additional caveats * * @return * @throws IOException * @throws org.exolab.castor.xml.MarshalException * @throws org.exolab.castor.xml.ValidationException * ????? where does this live JBPNote ? */ private VamsasDocument _doc=null; /** * Create a new vamsas archive * File locks are made immediately to avoid contention * * @param archive - file spec for new vamsas archive * @param vamsasdocument true if archive is to be a fully fledged vamsas document archive * @throws IOException if call to accessOriginal failed for updates, or openArchive failed. */ public VamsasArchive(File archive, boolean vamsasdocument) throws IOException { this(archive, false, vamsasdocument, null); } public VamsasArchive(File archive, boolean vamsasdocument, boolean overwrite) throws IOException { this(archive, overwrite, vamsasdocument, null); } /** * Constructor for accessing Files under file-lock management (ie a session file) * @param archive * @param vamsasdocument * @param overwrite * @throws IOException */ public VamsasArchive(VamsasFile archive, boolean vamsasdocument, boolean overwrite) throws IOException { this(archive.sessionFile, overwrite, vamsasdocument, archive); // log.debug("using non-functional lock-IO stream jar access constructor"); } /** * read and write to archive - will not overwrite original contents, and will always write an up to date vamsas document structure. * @param archive * @throws IOException */ public VamsasArchive(VamsasFile archive) throws IOException { this(archive, true, false); } /** * * @param archive file to write * @param overwrite true if original contents should be deleted * @param vamsasdocument true if a proper VamsasDocument archive is to be written. * @param extantLock SessionFile object holding a lock for the archive * @throws IOException */ public VamsasArchive(File archive, boolean overwrite, boolean vamsasdocument, SessionFile extantLock) throws IOException { super(); if (archive==null || (archive!=null && !(archive.getAbsoluteFile().getParentFile().canWrite() && (!archive.exists() || archive.canWrite())))) { log.fatal("Expect Badness! -- Invalid parameters for VamsasArchive constructor:"+((archive!=null) ? "File cannot be overwritten." : "Null Object not valid constructor parameter")); return; } this.vamsasdocument = vamsasdocument; if (archive.exists() && !overwrite) { this.original = archive; if (extantLock!=null) { this.odoclock = extantLock; if (odoclock.fileLock==null || !odoclock.fileLock.isLocked()) odoclock.lockFile(); } else { this.odoclock = new SessionFile(archive); } odoclock.lockFile(); // lock the file *immediatly* this.archive = null; // archive will be a temp file when the open method is called virginArchive=false; try { this.accessOriginal(); } catch (IOException e) { throw new IOException("Lock failed for existing archive"+archive); } } else { this.original = null; this.archive = archive; // archive is written in place. if (extantLock!=null) rchive=extantLock; else rchive = new SessionFile(archive); rchive.lockFile(); if (rchive.fileLock==null || !rchive.fileLock.isLocked()) throw new IOException("Lock failed for new archive"+archive); rchive.fileLock.getRaFile().setLength(0); // empty the archive. virginArchive = true; } this.openArchive(); // open archive } /** * open original archive file for exclusive (locked) reading. * @throws IOException */ private void accessOriginal() throws IOException { if (original!=null && original.exists()) { if (odoclock==null) odoclock = new SessionFile(original); odoclock.lockFile(); if (odoc == null) odoc = new VamsasArchiveReader(original); // this constructor is not implemented yet odoc = new VamsasArchiveReader(odoclock.fileLock); } } /** * Add unique entry strings to internal JarEntries list. * @param entry * @return true if entry was unique and was added. */ private boolean addEntry(String entry) { if (entries==null) entries=new Hashtable(); if (log.isDebugEnabled()) { log.debug("validating '"+entry+"' in hash for "+this); } if (entries.containsKey(entry)) return false; entries.put(entry, new Integer(entries.size())); return true; } /** * adds named entry to newarchive or returns false. * @param entry * @return true if entry was unique and could be added * @throws IOException if entry name was invalid or a new entry could not be made on newarchive */ private boolean addValidEntry(String entry) throws IOException { org.apache.tools.zip.ZipEntry je = new org.apache.tools.zip.ZipEntry(entry); // je.setExsetExtra(null); if (!addEntry(entry)) return false; newarchive.flush(); newarchive.putNextEntry(je); return true; } /** * called by app to get name of backup if it was made. * If this is called, the caller app *must* delete the backup themselves. * @return null or a valid file object */ public File backupFile() { if (!virginArchive) { makeBackup(); donotdeletebackup=true; // external reference has been made. return ((original!=null) ? originalBackup : null); } return null; } /** * Stops any current write to archive, and reverts to the backup if it exists. * All existing locks on the original will be released. All backup files are removed. */ public boolean cancelArchive() { if (newarchive!=null) { try { newarchive.closeEntry(); newarchive.putNextEntry(new org.apache.tools.zip.ZipEntry("deleted")); newarchive.closeEntry(); newarchive.close(); } catch (Exception e) { log.debug("Whilst closing newarchive",e); }; if (!virginArchive) { // then there is something to recover. try { recoverBackup(); } catch (Exception e) { log.warn("Problems when trying to cancel Archive "+archive.getAbsolutePath(), e); return false; } } } else { log.warn("Client Error: cancelArchive called before archive("+original.getAbsolutePath()+") has been opened!"); } closeAndReset(); // tidy up and release locks. return true; } /** * only do this if you want to destroy the current file output stream * */ private void closeAndReset() { if (rchive!=null) { rchive.unlockFile(); rchive=null; } if (original!=null) { if (odoc!=null) { odoc.close(); odoc=null; } if (archive!=null) archive.delete(); if (odoclock!=null) { odoclock.unlockFile(); odoclock = null; } } removeBackup(); newarchive=null; original=null; entries=null; } /** * Tidies up and closes archive, removing any backups that were created. * NOTE: It is up to the caller to delete the original archive backup obtained from backupFile() * TODO: ensure all extant AppDataReference jar entries are transferred to new Jar * TODO: provide convenient mechanism for generating new unique AppDataReferences and adding them to the document */ public void closeArchive() throws IOException { if (newarchive!=null) { newarchive.flush(); newarchive.closeEntry(); if (!isDocumentWritten()) log.warn("Premature closure of archive '"+archive.getAbsolutePath()+"': No document has been written."); newarchive.finish();// close(); // use newarchive.finish(); for a stream IO newarchive.flush(); // updateOriginal(); closeAndReset(); } else { log.warn("Attempt to close archive that has not been opened for writing."); } } /** * Opens and returns the applicationData output stream for the appdataReference string. * @param appdataReference * @return Output stream to write to * @throws IOException */ public AppDataOutputStream getAppDataStream(String appdataReference) throws IOException { if (newarchive==null) throw new IOException("Attempt to write to closed VamsasArchive object."); if (addValidEntry(appdataReference)) { return new AppDataOutputStream(newarchive); } return null; } /** * * @return JarEntry name for the vamsas XML stream in this archive */ protected String getDocumentJarEntry() { if (vamsasdocument) return VamsasArchiveReader.VAMSASDOC; return VamsasArchiveReader.VAMSASXML; } /** * Safely initializes the VAMSAS XML document Jar Entry. * @return Writer to pass to the marshalling function. * @throws IOException if a document entry has already been written. */ public PrintWriter getDocumentOutputStream() throws IOException { if (newarchive==null) openArchive(); if (!isDocumentWritten()) { try { if (addValidEntry(getDocumentJarEntry())) return new PrintWriter(new java.io.OutputStreamWriter(newarchive, "UTF-8")); } catch (Exception e) { log.warn("Problems opening XML document JarEntry stream",e); } } else { throw new IOException("Vamsas Document output stream is already written."); } return null; } /** * Access original archive if it exists, pass the reader to the client * Note: this is NOT thread safe and a call to closeArchive() will by necessity * close and invalidate the VamsasArchiveReader object. * @return null if no original archive exists. */ public VamsasArchiveReader getOriginalArchiveReader() throws IOException { if (!virginArchive) { accessOriginal(); return odoc; } return null; } /** * returns original document's root vamsas elements. * @return * @throws IOException * @throws org.exolab.castor.xml.MarshalException * @throws org.exolab.castor.xml.ValidationException */ public Vobject[] getOriginalRoots() throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { return VamsasArchive.getOriginalRoots(this); } /** * @return original document or a new empty document (with default provenance) * @throws IOException * @throws org.exolab.castor.xml.MarshalException * @throws org.exolab.castor.xml.ValidationException */ public VamsasDocument getVamsasDocument() throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { return getVamsasDocument("org.vamsas.simpleclient.VamsasArchive", "Created new empty document", null); } /** * Return the original document or a new empty document with initial provenance entry. * @param provenance_user (null sets user to be the class name) * @param provenance_action (null sets action to be 'created new document') * @param version (null means use latest version) * @return (original document or a new vamsas document with supplied provenance and version info) * @throws IOException * @throws org.exolab.castor.xml.MarshalException * @throws org.exolab.castor.xml.ValidationException */ public VamsasDocument getVamsasDocument(String provenance_user, String provenance_action, String version) throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { if (_doc!=null) return _doc; _doc = getOriginalVamsasDocument(this, getVorba()); if (_doc!=null) return _doc; // validate parameters if (provenance_user==null) provenance_user = "org.vamsas.simpleclient.VamsasArchive"; if (provenance_action == null) provenance_action="Created new empty document"; if (version==null) version = VersionEntries.latestVersion(); // Create a new document and return it _doc = DocumentStuff.newVamsasDocument(new VAMSAS[] { new VAMSAS()}, ProvenanceStuff.newProvenance(provenance_user, provenance_action), version); return _doc; } /** * @return Returns the current VorbaIdFactory for the archive. */ public VorbaIdFactory getVorba() { if (vorba==null) vorba = new SimpleDocument("simpleclient.VamsasArchive"); return vorba.getVorba(); } /** * @return true if Vamsas Document has been written to archive */ protected boolean isDocumentWritten() { if (newarchive==null) log.warn("isDocumentWritten() called for unopened archive."); if (entries!=null) { if (entries.containsKey(getDocumentJarEntry())) return true; } return false; } private void makeBackup() { if (!virginArchive) { if (originalBackup==null && original!=null && original.exists()) { try { accessOriginal(); originalBackup = odoclock.backupSessionFile(null, original.getName(), ".bak", original.getParentFile()); } catch (IOException e) { log.warn("Problem whilst making a backup of original archive.",e); } } } } /** * opens the new archive ready for writing. If the new archive is replacing an existing one, * then the existing archive will be locked, and the new archive written to a temporary file. * The new archive will be put in place once close() is called. * @param doclock LATER - pass existing lock on document, if it exists.... no need yet? * @throws IOException */ private void openArchive() throws IOException { if (newarchive!=null) { log.warn("openArchive() called multiple times."); throw new IOException("Vamsas Archive '"+archive.getAbsolutePath()+"' is already open."); } if (archive==null && (virginArchive || original==null)) { log.warn("openArchive called on uninitialised VamsasArchive object."); throw new IOException("Badly initialised VamsasArchive object - no archive file specified."); } if (!virginArchive) { // lock the original accessOriginal(); // make a temporary file to write to archive = File.createTempFile(original.getName(), ".new",original.getParentFile()); } else { if (archive.exists()) log.warn("New archive file name already in use! Possible lock failure imminent?"); } if (rchive==null) rchive = new SessionFile(archive); if (!rchive.lockFile()) throw new IOException("Failed to get lock on file "+archive); // LATER: locked IO stream based access. // Manifest newmanifest = new Manifest(); newarchive = new org.apache.tools.zip.ZipOutputStream(rchive.fileLock.getBufferedOutputStream(true));// , newmanifest); //newarchive = new JarOutputStream(new BufferedOutputStream(new java.io.FileOutputStream(archive))); entries = new Hashtable(); } public void putVamsasDocument(VamsasDocument doc) throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { putVamsasDocument(doc, getVorba()); } /** * * @param doc * @param vorba * @return (vorbaId string, Vobjhash) pairs for last hash of each object in document * @throws IOException * @throws org.exolab.castor.xml.MarshalException * @throws org.exolab.castor.xml.ValidationException */ public void putVamsasDocument(VamsasDocument doc, VorbaIdFactory vorba) throws IOException, org.exolab.castor.xml.MarshalException, org.exolab.castor.xml.ValidationException { if (vamsasdocument) doc.setVersion(VersionEntries.latestVersion()); // LATER: ensure this does the correct thing. VorbaXmlBinder.putVamsasDocument(getDocumentOutputStream(), vorba, doc); } /** * recovers the original file's contents from the (temporary) backup. * @throws Exception if any SessionFile or file removal operations fail. */ private void recoverBackup() throws Exception { if (originalBackup!=null) { // backup has been made. // revert from backup and delete it (changing backup filename) if (rchive==null) { rchive = new SessionFile(original); } SessionFile bckup = new SessionFile(originalBackup); rchive.updateFrom(null, bckup); // recover from backup file. bckup.unlockFile(); bckup=null; removeBackup(); } } /** * forget about any backup that was made - removing it first if it was only temporary. */ private void removeBackup() { if (originalBackup!=null) { log.debug("Removing backup in "+originalBackup.getAbsolutePath()); if (!donotdeletebackup) if (!originalBackup.delete()) log.info("VamsasArchive couldn't remove temporary backup "+originalBackup.getAbsolutePath()); originalBackup=null; } } /** * @param vorba the VorbaIdFactory to use for accessing vamsas objects. */ public void setVorba(VorbaIdFactory Vorba) { if (Vorba!=null) { if (vorba==null) vorba = new SimpleDocument(Vorba); else vorba.setVorba(Vorba); } else getVorba(); } /** * Convenience method to copy over the referred entry from the backup to the new version. * Warning messages are raised if no backup exists or the * entry doesn't exist in the backed-up original. * Duplicate writes return true - but a warning message will also be raised. * @param AppDataReference * @return true if AppDataReference now exists in the new document * @throws IOException */ public boolean transferAppDataEntry(String AppDataReference) throws IOException { return transferAppDataEntry(AppDataReference, AppDataReference); } /** * Validates the AppDataReference: not null and not already written to archive. * @param AppDataReference * @return true if valid. false if not * @throws IOException for really broken references! */ protected boolean _validNewAppDataReference(String newAppDataReference) throws IOException { // LATER: Specify valid AppDataReference form in all VamsasArchive handlers if (newAppDataReference==null) throw new IOException("null newAppDataReference!"); if (entries.containsKey(newAppDataReference)) { log.warn("Attempt to write '"+newAppDataReference+"' twice! - IGNORED"); // LATER: fix me? warning message should raise an exception here. return false; } return true; } /** * Transfers an AppDataReference from old to new vamsas archive, with a name change. * @see transferAppDataEntry(String AppDataReference) * @param AppDataReference * @param NewAppDataReference - AppDataReference in new Archive * @return * @throws IOException */ public boolean transferAppDataEntry(String AppDataReference, String NewAppDataReference) throws IOException { if (original==null || !original.exists()) { log.warn("No backup archive exists."); return false; } if (AppDataReference==null) throw new IOException("null AppDataReference!"); if (!_validNewAppDataReference(NewAppDataReference)) return false; accessOriginal(); java.io.InputStream adstream = odoc.getAppdataStream(AppDataReference); if (adstream==null) { log.warn("AppDataReference '"+AppDataReference+"' doesn't exist in backup archive."); return false; } java.io.OutputStream adout = getAppDataStream(NewAppDataReference); // copy over the bytes int written=-1; long count=0; byte[] buffer = new byte[_TRANSFER_BUFFER]; // conservative estimate of a sensible buffer do { if ((written = adstream.read(buffer))>-1) { adout.write(buffer, 0, written); log.debug("Transferring "+written+"."); count+=written; } } while (written>-1); log.debug("Sucessfully transferred AppData for '" +AppDataReference+"' as '"+NewAppDataReference+"' ("+count+" bytes)"); return true; } /** * write data from a stream into an appData reference. * @param AppDataReference - New AppDataReference not already written to archive * @param adstream Source of data for appData reference - read until .read(buffer) returns -1 * @return true on success. * @throws IOException for file IO or invalid AppDataReference string */ public boolean writeAppdataFromStream(String AppDataReference, java.io.InputStream adstream) throws IOException { if (!_validNewAppDataReference(AppDataReference)) { log.warn("Invalid AppDataReference passed to writeAppdataFromStream"); throw new IOException("Invalid AppDataReference! (null, or maybe non-unique)!"); } if (AppDataReference==null) { log.warn("null appdata passed."); throw new IOException("Null AppDataReference"); } java.io.OutputStream adout = getAppDataStream(AppDataReference); // copy over the bytes int written=-1; long count=0; byte[] buffer = new byte[_TRANSFER_BUFFER]; // conservative estimate of a sensible buffer do { if ((written = adstream.read(buffer))>-1) { adout.write(buffer, 0, written); log.debug("Transferring "+written+"."); count+=written; } } while (written>-1); return true; } /** * transfers any AppDataReferences existing in the old document * that haven't already been transferred to the new one * LATER: do the same for transfers requiring a namechange - more document dependent. * @return true if data was transferred. */ public boolean transferRemainingAppDatas() throws IOException { boolean transfered=false; if (original==null || !original.exists()) { log.warn("No backup archive exists."); return false; } accessOriginal(); if (getVorba()!=null) { Vector originalRefs=null; try { originalRefs = vorba.getReferencedEntries(getVamsasDocument(), getOriginalArchiveReader()); } catch (Exception e) { log.warn("Problems accessing original document entries!",e); } if (originalRefs!=null) { Iterator ref = originalRefs.iterator(); while (ref.hasNext()) { String oldentry = (String) ref.next(); if (oldentry!=null && !entries.containsKey(oldentry)) { log.debug("Transferring remaining entry '"+oldentry+"'"); transfered |= transferAppDataEntry(oldentry); } } } } return transfered; } /** * called after archive is written to put file in its final place */ private void updateOriginal() { if (!virginArchive) { // make sure original document really is backed up and then overwrite it. if (odoc!=null) { // try to shut the odoc reader. odoc.close(); odoc = null; } // Make a backup if it isn't done already makeBackup(); try { // copy new Archive data that was writen to a temporary file odoclock.updateFrom(null, rchive); } catch (IOException e) { // LATER: decide if leaving nastily named backup files around is necessary. File backupFile=backupFile(); if (backupFile!=null) log.error("Problem updating archive from temporary file! - backup left in '" +backupFile().getAbsolutePath()+"'",e); else log.error("Problems updating, and failed to even make a backup file. Ooops!", e); } // Tidy up if necessary. removeBackup(); } else { } } }