C#

<br />
 public virtual bool TransformXML2( string strTempXml, ref string strTransformedXml )<br />
        {<br />
            bool bSuccess = true;<br />
            if (m_strTransformFile.Length == 0)<br />
            {<br />
                PIUUtilitiesMgr.PIUMessage(PIUUtilitiesMgr.GetResourceString("MISSING_XSLT_FILE_NAME"), PIUUtilitiesMgr.GetResourceString("PERSONNEL_IMPORT_UTILITY"), MessageBoxButtons.OK);<br />
                return false;<br />
            }<br />
<br />
            //Create Dom object and transform with XSL file<br />
            try<br />
            {<br />
<br />
                System.IO.MemoryStream streamTempData = new System.IO.MemoryStream();<br />
                PIUUtilitiesMgr.FromStringToMemoryStream(streamTempData, strTempXml);<br />
                string outText = PIUUtilitiesMgr.MemoryStreamToString(streamTempData);<br />
<br />
                PIUUtilitiesMgr.WriteToFileForDebuggingPurpose("TempUserList.xml", outText, true);<br />
<br />
                //Transform<br />
                System.IO.MemoryStream streamTransformedData = new System.IO.MemoryStream();<br />
                //Create a reader to read the stream<br />
                streamTempData.Position = 0;<br />
<br />
<br />
                XmlReader reader = XmlReader.Create(streamTempData);<br />
                XslCompiledTransform xslTransform = new XslCompiledTransform(true);<br />
<br />
                XsltSettings settings = new XsltSettings(true, true);<br />
                //xslTransform.Load(m_strTransformFile);<br />
                xslTransform.Load(m_strTransformFile, settings, new XmlUrlResolver());<br />
<br />
                xslTransform.Transform(reader, null, streamTransformedData);<br />
<br />
                //set position to 0<br />
                streamTransformedData.Position = 0;<br />
<br />
                //Convert this to string<br />
                strTransformedXml = PIUUtilitiesMgr.MemoryStreamToString(streamTransformedData);<br />
<br />
                // we get ??? at the top of the file and need to get rid of it.<br />
                strTransformedXml = strTransformedXml.Remove(0, 1);<br />
                streamTransformedData.Close();<br />
                streamTransformedData.Dispose();<br />
                streamTempData.Close();<br />
                streamTempData.Dispose();<br />
<br />
<br />
<br />
                //If debug write this to file<br />
               <br />
                if (piuInfo.DebugMode)<br />
                {<br />
                    lock (this)<br />
                    {<br />
                        PIUUtilitiesMgr.WriteToFileForDe1buggingPurpose("TransformedUserList.xml", strTransformedXml, true);<br />
                    }<br />
                }<br />
<br />
            }<br />
<br />

        // 2. Start DataGridView "dataGridView2". <br />
        // 2a. create a mysql DataAdapter<br />
        OdbcAd = new System.Data.Odbc.OdbcDataAdapter("SELECT * FROM " + SelectedTableName + ";", OdbcCon);<br />
<br />
        // 2b. create a dataset<br />
        DataSet myds = new DataSet();<br />
<br />
        // 2c. now fill and bind the DataGrid<br />
        OdbcAd.Fill(myds, SelectedTableName);<br />
<br />
        // 2d. dataGridView1.DataSource = myds;<br />
        dataGridView2.DataSource = myds.Tables[SelectedTableName];

...
...
//deflate and chunk
object[] outData = DeflateAndChunk(data);
...
...
//dechunk and inflate
string reconstructedData = DechunkAndInflate(outData);
...
...

private static byte[] DeChunkData(object[] baseData)
{
	int returnLength = 0;
	foreach(byte[] ba in baseData)
	{
		returnLength += ba.Length;
	}

	byte[] readBuffer = new byte[returnLength];
	using (Stream outStream = new MemoryStream(readBuffer))
	{
		
		for (int loop=0; loop<baseData.Length; loop++)
		{
			byte[] dataChunk = (byte[])baseData[loop];
			outStream.Write(dataChunk, 0, dataChunk.Length);	
			baseData[loop] = null; //null of the bytes that are no longer needed, every little helps
		}


		outStream.Flush();
		outStream.Close();
	}

	GC.Collect();

	return readBuffer;
}

private static object[] ChunkData(byte[] baseData, int chunkSize)
{
	int returnArraySize = baseData.Length / chunkSize;
	int baseDataModulus = baseData.Length % chunkSize;
	if (baseDataModulus > 0)
	{
		returnArraySize++;
	}
	object[] returnData = new object[returnArraySize];

	using (Stream baseDataStream = new MemoryStream(baseData))
	{
		baseData = null;

		for (int loop=0;loop < returnArraySize; loop++)
		{
			int workingChunkSize = chunkSize;

			if (loop == returnArraySize)
			{
				workingChunkSize = baseDataModulus;
			}

			byte[] buffer = new byte[workingChunkSize];
			baseDataStream.Read(buffer, 0, workingChunkSize);
			returnData[loop] = buffer;
		}

		baseDataStream.Flush();
		baseDataStream.Close();
	}

	GC.Collect();

	return returnData;
}

public static object[] DeflateAndChunk(string inputMessage)
{
	//compress data and get checksum
	long adler32Checksum = 0;
	byte[] compressedData = CompressData(inputMessage, out adler32Checksum);

	inputMessage = null;
	
	//TODO - BEGIN - Define chunk sizes (config? Based on data size?)
	int chunkSize = 1000;
	//TODO - END Define chunk sizes (config? Based on data size?)

	//chunk data based on params
	object[] chunkedData = ChunkData(compressedData, chunkSize);

	//TODO - BEGIN - thread off and send chunks using config from WS call
	//WS calls here, least of my worries until memory
	//usage for chunking and compression is sorted
	//TODO - END - thread off and send chunks using config from WS call

	//method only returns so it can be tested
	return chunkedData;
}

public static string DechunkAndInflate(object[] inputData)
{
	byte[] deChunkedData = DeChunkData(inputData);

	byte[] inflatedData = UnCompressData(deChunkedData);
	
	deChunkedData = null;

	return System.Text.ASCIIEncoding.ASCII.GetString(inflatedData);
}

private static byte[] CompressData(string message, out long adler32Checksum)
{
	byte[] returnData = null;

	//create base streams
	using (Stream inStream = new MemoryStream(System.Text.ASCIIEncoding.ASCII.GetBytes(message)))
	{
		message = null;

		using (Stream outStream = new MemoryStream())
		using (ZipStreams.DeflaterOutputStream deflatorStream = new ZipStreams.DeflaterOutputStream(outStream, 
			new ZipCompression.Deflater(ZipCompression.Deflater.BEST_SPEED)))
		{

			....
			//standard sharpzip code for compression
			....
		}
	}

	//calculate checksums
	BaseCompression.Checksums.Adler32 ad32 = new BaseCompression.Checksums.Adler32();
	ad32.Update(returnData);
	adler32Checksum = ad32.Value;

	//force garbage collection
	GC.Collect();

	return returnData;
}

private static byte[] UnCompressData(byte[] deflatedData)
{
	byte[] returnData = null;

	//create inflator streams
	using (Stream inStream = new MemoryStream(deflatedData))
	{
		deflatedData = null;

		using (MemoryStream outStream = new MemoryStream())
		using (ZipStreams.InflaterInputStream inflatorStream = new ZipStreams.InflaterInputStream(inStream))
		{
			....
			//standard sharpzip code for uncompression
			....
		}
	}

	//force garbage collection
	GC.Collect();

	//return
	return returnData;
}