<?xml version="1.0" encoding="utf-8"?>
<!--

Copyright (c) 2002-2004 Regenstrief Institute.  All rights reserved.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

Written by Gunther Schadow.

$Id: reportdump-sectioner.xsl,v 1.5 2005/02/22 00:25:50 gschadow Exp $

-->
<xsl:transform version="1.0"
   xmlns:saxon="http://saxon.sf.net/"
   xmlns:xwsf="java:net.sf.xwsf.saxon.Functions"
   xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
   xmlns:xs="http://www.w3.org/2001/XMLSchema"
   xmlns:ex="http://xwsf.sf.net/net.sf.xwsf.saxon.ExceptionElementFactory"
   xmlns:exd="http://xwsf.sf.net/exception"
   xmlns:rs="java:java.sql.ResultSet"
   xmlns:sys="java:java.lang.System"
   xmlns:gut="http://aurora.regenstrief.org/XML/GenericUpTranslator"
   xmlns:gst="http://xwsf.sf.net/gstserver/"
   xmlns:d="http://deid.upmc.edu/"
   extension-element-prefixes="saxon ex sys"
   exclude-result-prefixes="saxon xwsf ex rs exd sys gut gst xs d">
<!-- reportdump

  Read text reports from the data base, split them into sections,
  scrub them and load them back into the database in a different field. Later
  they would also be scrubbed.
-->

  <xsl:import href="exception.xsl"/>
  <xsl:import href="sectioner.xsl"/>
  <xsl:import href="db-common.xsl"/>
  <xsl:import href="deid2plain.xsl"/>
  <xsl:import href="scrubber.xsl"/>
 
  <xsl:output method="xml" 
	      indent="no" 
	      encoding="UTF-8"
	      media-type="text/xml"/>
  <xsl:output name="default" 
              method="xml" 
	      indent="no" 
	      omit-xml-declaration="yes"
	      encoding="UTF-8"
	      media-type="text/xml"/>
  <xsl:strip-space elements="*"/>

  <xsl:param name="debug" select="0" as="xs:integer"/>
  <xsl:param name="check" select="/*/@check"/>
  <xsl:param name="force" select="/*/@force"/>
  <xsl:param name="use-clob" select="/*/@use-clob"/>
  <xsl:param name="immediate" select="/.."/>
  <xsl:param name="prequeue" select="/.."/>
  <xsl:param name="scrub" select="'yes'"/>
  <xsl:param name="modulo" select="/.."/>

  <xsl:param name="reportSQL">
     SELECT --+FIRST_ROWS
            M.INSTITUTION_ID,
            M.MULTIMEDIA_CONTENT_ID,
	    O.SERVICE_CODE,
	    O.SERVICE_SYS_ID,
	    -- O.PHYSIOLOGIC_TIME,
            M.MIME_TYPE_CODE,
            M.MULTIMEDIA_TEXT_CONTENT,
	    <xsl:if test="$prequeue">Q.ROWID AS QUEUE_ROWID,</xsl:if>
	    M.PATIENT_ID
       FROM <xsl:if test="$prequeue"><xsl:value-of select="$prequeue"/> Q
         INNER JOIN</xsl:if> RMRS.MULTIMEDIA_CONTENT M<xsl:if test="$prequeue">
	   ON(    Q.MULTIMEDIA_CONTENT_ID=M.MULTIMEDIA_CONTENT_ID
	      AND Q.INSTITUTION_ID=M.INSTITUTION_ID)</xsl:if>
	 INNER JOIN RMRS.MEDICAL_ORDER O
           ON(    O.MEDICAL_ORDER_ID=M.MEDICAL_ORDER_ID
	      AND O.INSTITUTION_ID=M.INSTITUTION_ID)
  </xsl:param>

  <xsl:param name="namesSQL">
    SELECT NAME_FIRST,
            NAME_LAST
       FROM RMRS.PATIENT_ALIAS_NAME
       WHERE (INSTITUTION_ID = ? AND PATIENT_ID = ?)
   UNION ALL
     SELECT NAME_FIRST,
            NAME_LAST
       FROM RMRS.PATIENT
       WHERE (INSTITUTION_ID = ? AND PATIENT_ID = ?)
  </xsl:param>
  <xsl:param name="namesStmt"
       xmlns:cn="java:java.sql.Connection"
       select="cn:prepareStatement($connection, $namesSQL)"/>

  <xsl:param name="checkSQL">
     UPDATE MULTIMEDIA_CONTENT
        SET MULTIMEDIA_TEXT_CONTENT_SCRUB=?
      WHERE INSTITUTION_ID=?
        AND MULTIMEDIA_CONTENT_ID=?
  </xsl:param>
  <xsl:param name="checkStmt"
       xmlns:cn="java:java.sql.Connection"
       select="cn:prepareStatement($connection, $checkSQL)"/>

  <xsl:param name="clob" xmlns:c="java:ClobMaker"
       select="c:make($connection)"/>  

  <xsl:param name="dequeueSQL">
     DELETE FROM <xsl:value-of select="$prequeue"/>
      WHERE ROWID=?
  </xsl:param>
  <xsl:param name="dequeueStmt"
       xmlns:cn="java:java.sql.Connection"
       select="cn:prepareStatement($connection, $dequeueSQL)"/>

  <xsl:template match="report-batch">
    <xsl:message>
      <start connection="{$driversLoaded}" check="{$check}" scrub="{if($scrub ne 'no') then 'yes' else 'no'}"/>
    </xsl:message>
    <ex:try atomic="no">
      <xsl:choose>
        <xsl:when test="$immediate">
	  <xsl:message><IMMEDIATE id="{$immediate}"/></xsl:message>
	  <xsl:variable name="fake"><report/></xsl:variable>
          <xsl:apply-templates mode="sqlselect" select="$fake">
            <xsl:with-param name="whereClause"
              select="concat(' WHERE M.MULTIMEDIA_CONTENT_ID = ',$immediate)"/>
          </xsl:apply-templates>
        </xsl:when>
        <xsl:otherwise>
          <xsl:apply-templates select="node()"/>
        </xsl:otherwise>
      </xsl:choose>
      <xsl:message>
        <end/>
      </xsl:message>

      <ex:catch exception="java.lang.Throwable">
        <xsl:message terminate="yes">
	  <xsl:call-template name="exd:dump">
	    <xsl:with-param name="name" select="'ERROR'"/>
	  </xsl:call-template>
        </xsl:message>
      </ex:catch>
      <ex:finally>
        <xsl:sequence xmlns:c="java:java.sql.Connection"
                      select="c:close($connection)"/>
      </ex:finally>
    </ex:try>  
    <xsl:message>
      <out/>
    </xsl:message>
  </xsl:template>

  <xsl:template match="/|@*|node()">
    <xsl:apply-templates select="@*|node()"/>
  </xsl:template>

  <xsl:template match="report">
    <xsl:param name="whereClause">
      <xsl:if test="not($force)"> 
        AND MULTIMEDIA_TEXT_CONTENT_SCRUB IS NULL
      </xsl:if>
      <xsl:if test="$modulo and $modulo ne '' and $modulo ne '-1'">
	<xsl:text>AND MOD(M.MULTIMEDIA_CONTENT_ID,10) IN (</xsl:text><xsl:value-of select="$modulo"/><xsl:text>)</xsl:text>
      </xsl:if>
      <xsl:apply-templates mode="makeWhereClause" select="@*|node()"/>
    </xsl:param>
    <xsl:choose>
      <xsl:when test="$whereClause">
        <xsl:apply-templates mode="sqlselect" select=".">
          <xsl:with-param name="whereClause"
	    select="concat(' WHERE',substring-after($whereClause, 'AND'))"/>
        </xsl:apply-templates>
      </xsl:when>
      <xsl:otherwise>
        <xsl:apply-templates mode="sqlselect" select="."/>
      </xsl:otherwise>
    </xsl:choose>
  </xsl:template>

  <!-- FOR TEST PURPOSES ONLY -->
  <xsl:template match="text_report">
    <xsl:message>
      <start connection="{$driversLoaded}" check="{$check}" scrub="{if($scrub ne 'no') then 'yes' else 'no'}"/>
    </xsl:message>
    <xsl:variable name="plainText">
      <xsl:variable name="strippedText">
        <xsl:apply-templates mode="strip-tags" select="."/>
      </xsl:variable>
      <xsl:sequence select="string-join($strippedText,'')"/>
    </xsl:variable>
    <!-- xsl:variable name="result">
      <xsl:apply-templates mode="gut:top" select="$plainText/text()"/>
    </xsl:variable -->
    <xsl:variable name="result">
      <xsl:apply-templates mode="gut:top" select="$plainText/text()"/>
    </xsl:variable>
    <xsl:variable name="scrubbedResult">
      <xsl:choose>
        <xsl:when test="$scrub ne 'no'">
          <xsl:apply-templates mode="scrub" select="$result"/>
        </xsl:when>
        <xsl:otherwise>
          <xsl:sequence select="$result"/>
        </xsl:otherwise>
      </xsl:choose>
    </xsl:variable>
    <xsl:variable name="finalResult">
      <xsl:apply-templates mode="finalize" select="$scrubbedResult"/>
    </xsl:variable>
    <TEST>
      <INPUT>
        <xsl:sequence select="."/>
      </INPUT>
      <PLAIN>
        <xsl:sequence select="$plainText"/>
      </PLAIN>
      <RESULT>
        <xsl:sequence select="$result"/>
      </RESULT>
      <SCRUBBED>
        <xsl:sequence select="$finalResult"/>
      </SCRUBBED>
    </TEST>
  </xsl:template>

  <!-- MODE: sqlselect - do the actual work -->

  <xsl:template mode="sqlselect" match="report">
    <xsl:param name="whereClause" select="''"/>
    <xsl:if test="$debug > 1">
      <xsl:message>
	<reportSQL>
          <xsl:value-of select="concat($reportSQL,$whereClause)"/>
	</reportSQL>
      </xsl:message>
    </xsl:if>
    <xsl:variable name="rset"
       xmlns:cn="java:java.sql.Connection"
       xmlns:st="java:java.sql.Statement"
       select="st:executeQuery(cn:createStatement($connection), 
                     concat($reportSQL,$whereClause))"/>
    <xsl:if test="$debug > 3">
      <xsl:call-template name="showMetaData">
	<xsl:with-param name="rset" select="$rset"/>
      </xsl:call-template>
    </xsl:if>

    <saxon:while test="rs:next($rset)">
      <xsl:variable name="id" select="rs:getLong($rset,'MULTIMEDIA_CONTENT_ID')"/>
      <xsl:variable name="institutionId" select="rs:getLong($rset,'INSTITUTION_ID')"/>
      <xsl:variable name="patientId" select="rs:getLong($rset,'PATIENT_ID')"/>
      <xsl:variable name="serviceCode" select="rs:getString($rset,'SERVICE_CODE')"/>
      <xsl:variable name="queueId" select="if($prequeue) then rs:getString($rset,'QUEUE_ROWID') else /.."/>
      <xsl:message>
        <ID id="{$id}" institutionId="{$institutionId}" serviceCode="{$serviceCode}"/>
      </xsl:message>
      <xsl:variable name="result">
        <ex:try atomic="yes">
	  <xsl:variable name="textString">
	    <xsl:choose>
	      <xsl:when test="$use-clob='no'">
		<xsl:value-of 
		   select="rs:getString($rset,'MULTIMEDIA_TEXT_CONTENT')"/>
	      </xsl:when>
	      <xsl:otherwise>
		<xsl:variable name="textClob"
		   select="rs:getClob($rset,'MULTIMEDIA_TEXT_CONTENT')"/>       
		<xsl:value-of xmlns:clob="java:java.sql.Clob"
		   select="clob:getSubString($textClob,1,
					     clob:length($textClob))"/>
	      </xsl:otherwise>
	    </xsl:choose>
	  </xsl:variable>
	  <xsl:variable name="text">
	    <ex:try atomic="yes">
	      <xsl:apply-templates mode="strip-tags" select="saxon:parse($textString)"/>

	      <ex:catch exception="java.lang.Throwable">
		<xsl:if test="$debug > 0">
		  <xsl:message>
		    <WARNING exception="{ex:current-exception()}"/>
		  </xsl:message>
		</xsl:if>
		<ex:try atomic="yes" xmlns:t="java:org.regenstrief.saxon.tidy">
		  <xsl:variable name="tidy-text" 
		    select="t:tidy(rs:getAsciiStream($rset,'MULTIMEDIA_TEXT_CONTENT'))"/>
		  <xsl:apply-templates mode="strip-tags" select="saxon:parse($tidy-text)"/>

		  <ex:catch exception="java.lang.Throwable">
		    <xsl:message>
		      <WARNING id="{$id}" exception="{ex:current-exception()}"/>
		    </xsl:message>
		  </ex:catch>
		</ex:try>
	      </ex:catch>
	    </ex:try>
	  </xsl:variable>
	  <xsl:if test="$text/text()">
	    <xsl:variable name="string"> 
	      <xsl:value-of select="string-join($text/text(),'')"/>
	    </xsl:variable>  
            <xsl:variable name="text-to-parse" select="$string"/>
            <xsl:if test="$debug > 3">
              <xsl:message>
   	        <TEXT>
                  <xsl:value-of select="$string"/>
	        </TEXT>
              </xsl:message>
            </xsl:if>
	    <text_report id="{$id}" institutionId="{$institutionId}">
              <xsl:variable name="namesResultSet"
                  xmlns:st="java:java.sql.PreparedStatement"
                  select="xwsf:progn(
			   st:setLong($namesStmt, 1, $institutionId),
                           st:setLong($namesStmt, 2, $patientId),
			   st:setLong($namesStmt, 3, $institutionId),
                           st:setLong($namesStmt, 4, $patientId),
                           st:executeQuery($namesStmt))"/>
              <saxon:while test="rs:next($namesResultSet)">
                <scrubWords>
                  <word>
 	            <xsl:value-of select="rs:getString($namesResultSet,1)"/>
                  </word>
                  <word>
 	            <xsl:value-of select="rs:getString($namesResultSet,2)"/>
                  </word>
                </scrubWords>
              </saxon:while>
	      <xsl:apply-templates mode="gut:top" select="$text-to-parse/text()"/>
	    </text_report>
	  </xsl:if>
	  <ex:catch exception="org.regenstrief.util.ProtectedMatcher$TimeoutException">
	    <timeout>
              <xsl:value-of select="ex:current-exception()"/>
            </timeout>
	  </ex:catch>
	  <ex:catch exception="java.lang.Throwable">
            <xsl:call-template name="exd:dump">
	      <xsl:with-param name="name" select="exception"/>
	    </xsl:call-template>
	  </ex:catch>
	</ex:try>
      </xsl:variable>
      <xsl:variable name="scrubbedResult">
         <xsl:choose>
           <xsl:when test="$scrub ne 'no'">
	     <xsl:apply-templates mode="scrub" select="$result"/>
	   </xsl:when>
	  <xsl:otherwise>
	    <xsl:sequence select="$result"/>
	  </xsl:otherwise>
        </xsl:choose>
      </xsl:variable>
      <xsl:variable name="finalResult">
        <xsl:apply-templates mode="finalize" select="$scrubbedResult"/>
      </xsl:variable>
      <xsl:if test="$debug>4">
        <xsl:message>
          <RESULT>
            <xsl:sequence select="$finalResult"/>
	  </RESULT>
        </xsl:message>
      </xsl:if>

      <xsl:if test="$result/text_report and $check">
	<xsl:variable name="clob-writer"
	   xmlns:clob="java:oracle.sql.CLOB"
	   select="xwsf:progn(clob:open($clob,clob:MODE_READWRITE()),
		    clob:trim($clob,0),
		    clob:getCharacterOutputStream($clob))"/>
	<xsl:variable name="clob-written"
	   xmlns:writer="java:java.io.Writer"
	   xmlns:clob="java:oracle.sql.CLOB"
	   select="xwsf:progn(writer:write($clob-writer,
		   saxon:serialize($finalResult,'default')),
		   writer:flush($clob-writer),
		   writer:close($clob-writer),
		   clob:close($clob),
		   $clob)"/>
	<xsl:variable name="checked"
	    xmlns:st="java:java.sql.PreparedStatement"
	    xmlns:c="java:java.sql.Connection"
	    xmlns:ts="java:java.sql.Timestamp"
	    select="xwsf:progn(if($queueId) 
		     then ( st:setString($dequeueStmt,1,$queueId),
			    st:executeUpdate($dequeueStmt) )
		     else /..,
		     st:setLong($checkStmt, 2, $institutionId),
	             st:setLong($checkStmt, 3, $id),
		     st:setClob($checkStmt, 1, $clob-written),
		     st:executeUpdate($checkStmt))"/>
	<xsl:message>
	  <checked value="{$checked}" id="{$id}" institutionId="{$institutionId}"/>
	</xsl:message>
      </xsl:if>
    </saxon:while>
  </xsl:template>

<!-- MODE: makeWhereClause -->

<xsl:template mode="makeWhereClause" match="report/@id">
  <xsl:text> AND M.MULTIMEDIA_CONTENT_ID = </xsl:text>
  <xsl:value-of select="."/>
</xsl:template>

<xsl:template mode="makeWhereClause" match="report/@code">
  <xsl:text> AND (O.SERVICE_SYS_ID=1 AND O.SERVICE_CODE IN (''</xsl:text>
  <xsl:for-each select="tokenize(., '\s+')">
     <xsl:text>, '</xsl:text>
     <xsl:value-of select="."/>
     <xsl:text>'</xsl:text>
  </xsl:for-each>
  <xsl:text>) )</xsl:text>
</xsl:template>

<xsl:template mode="makeWhereClause" match="report/@patientId">
  <xsl:text> AND M.PATIENT_ID = </xsl:text>
  <xsl:value-of select="."/>
</xsl:template>

<xsl:template mode="makeWhereClause" match="report/@orderId">
  <xsl:text> AND M.MEDICAL_ORDER_ID = </xsl:text>
  <xsl:value-of select="."/>
</xsl:template>

<xsl:template mode="makeWhereClause" match="report/@institutionId">
  <xsl:text> AND M.INSTITUTION_ID = </xsl:text>
  <xsl:value-of select="."/>
</xsl:template>

<xsl:template mode="makeWhereClause" match="report/@where">
  <xsl:text> AND </xsl:text>
  <xsl:value-of select="."/>
</xsl:template>

<xsl:template mode="makeWhereClause" match="/|@*|node()">
  <xsl:apply-templates mode="makeWhereClause" select="@*|node()"/>
</xsl:template>


<!-- MODE: text -->

<xsl:template mode="text" match="/|@*|node()">
  <xsl:copy>
    <xsl:apply-templates mode="text" select="@*|node()"/>
  </xsl:copy>
</xsl:template>

<xsl:template mode="text" match="text_report">
  <text>
    <xsl:apply-templates mode="text" select="@*|node()"/>    
  </text>
</xsl:template>

<xsl:template mode="text" match="text">
  <p>
    <xsl:apply-templates mode="text" select="@*|node()"/>    
  </p>
</xsl:template>

<xsl:template name="showMetaData">
  <xsl:param name="rset" select="/.."/>

  <xsl:message>
    <xsl:variable name="i" select="1" saxon:assignable="yes"/>
    <xsl:variable name="metaData" select="rs:getMetaData($rset)"/>
    <xsl:variable name="max" 
       xmlns:rm="java:java.sql.ResultSetMetaData"
       select="rm:getColumnCount($metaData)"/>
    <saxon:while test="$i &lt;= $max">
      <column
         xmlns:rm="java:java.sql.ResultSetMetaData"
         name="{rm:getColumnName($metaData,$i)}"
         type="{rm:getColumnType($metaData,$i)}"
         schemaName="{rm:getSchemaName($metaData,$i)}"
         tableName="{rm:getTableName($metaData,$i)}"
      />
      <saxon:assign name="i" select="$i + 1"/> 
    </saxon:while>
  </xsl:message>
</xsl:template>

<!-- MODE: strip-tags: remove all layout tags and render everything in
       plain text.  This is a deep null-transform that only catches
       text nodes and some handling of paragraphs, text_report,
       text/@title etc.  
-->
<xsl:template mode="strip-tags" match="/|node()|@*">
  <xsl:apply-templates mode="strip-tags" select="@*|node()"/>
</xsl:template>

<xsl:template mode="strip-tags" match="text()" priority="0">
  <xsl:value-of select="current()"/>
</xsl:template>

<xsl:template mode="strip-tags" match="d:*" priority="0">
  <xsl:apply-templates mode="deid2plain" select="current()"/>
</xsl:template>

<xsl:template mode="strip-tags" 
	      match="text[@title and starts-with(text()[1],@title)]"
	      priority="3">
  <xsl:value-of select="'&#xA;&#xA;'"/>
  <xsl:value-of select="@title"/>
  <xsl:value-of select="':&#xA;'"/>
  <xsl:value-of 
       select="substring-after(substring-after(text()[1],@title),':')"/>
  <xsl:message><text title="{@title}"/></xsl:message>
  <xsl:apply-templates mode="strip-tags" select="@*|node() except text()[1]"/>
  <!-- xsl:value-of select="'&#xA;'"/-->
</xsl:template>

<xsl:template mode="strip-tags" match="text[@title]">
  <xsl:value-of select="'&#xA;&#xA;'"/>
  <xsl:value-of select="@title"/>
  <xsl:value-of select="':&#xA;'"/>
  <xsl:message><text title="{@title}"/></xsl:message>
  <xsl:apply-templates mode="strip-tags" select="@*|node()"/>
  <xsl:value-of select="'&#xA;&#xA;'"/>
</xsl:template>

<xsl:template mode="strip-tags" match="p">
  <xsl:variable name="priorThing" select="(preceding-sibling::*|preceding-sibling::text())[last()]"/>
  <xsl:variable name="nextThing" select="(following-sibling::*|following-sibling::node())[1]"/>
  <xsl:if test="$priorThing
            and not($priorThing/self::p)
            and not(ends-with($priorThing/self::text(),'&#xA;'))">
    <xsl:value-of select="'&#xA;'"/>
  </xsl:if>
  <xsl:value-of select="'&#xA;'"/>
  <xsl:apply-templates mode="strip-tags" select="@*|node()"/>
  <xsl:value-of select="'&#xA;'"/>
  <xsl:if test="$nextThing
            and not($nextThing/self::p)
            and not(ends-with($nextThing/self::text(),'&#xA;'))">
    <xsl:value-of select="'&#xA;'"/>
  </xsl:if>
</xsl:template>

<xsl:template mode="strip-tags" match="br[not(starts-with(following-sibling::text()[1],'&#xA;'))]">
  <xsl:value-of select="'&#xA;'"/>
</xsl:template>

<!-- MODE: RENDER FOR TEXTINDEX -->
<xsl:template mode="finalize" match="/|@*|node()">
  <xsl:copy>
    <xsl:apply-templates mode="finalize" select="@*|node()"/>
  </xsl:copy>
</xsl:template>
<xsl:template mode="finalize" match="@title">
  <title><xsl:value-of select="."/></title>
</xsl:template>

</xsl:transform>
