#!/usr/bin/perl
# -----------------------------------------------------------------------------
# TWC
# Treuwert-Computer GmbH
# Schloßbergring 9
# 79098 Freiburg
#
# -----------------------------------------------------------------------------
# Programm-Daten
# -----------------------------------------------------------------------------
# Projekt  : Tools
# System   : Linux
# Programm : ImportInDB.pl
# Modul    : $RCSfile: InstallDB.pl,v $
# Version  : $Revision: 1.10 $
# Stand von: $Date: 2002/03/18 14:36:27 $
# Status   : $State: Exp $
#
# Description:
# This Script is used, to maintain your WWWdb-database-tables
#
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# $Log: InstallDB.pl,v $
# Revision 1.10  2002/03/18 14:36:27  k_reger
# - PostgreSQL-type correction
#
# Revision 1.9  2002/03/12 08:34:48  k_reger
# - new datatype-mapping
#
# Revision 1.8  2002/02/28 14:59:24  k_reger
# - Unnecessary modules removed
#
# Revision 1.7  2002/02/27 08:34:39  k_reger
# - no more "use lib"
#
# Revision 1.6  2001/01/25 09:43:59  k_reger
# - new db-authentification-switsches
# - new switch -I for regenerating indices
# - new switch -W for passing where-clauses
# - InterBase and ODBC-Support
#
# Revision 1.5  2000/11/30 15:22:25  k_reger
# - patch from Pirmin Kalberer
#
# Revision 1.4  2000/10/12 13:49:34  k_reger
# - -I re-creates the indices
#
# Revision 1.3  2000/03/24 09:54:57  klaus
# Checkpoint
#
# Revision 1.2  2000/02/24 11:27:13  klaus
# - Internationalization
# - New OO-Classes
#
# Revision 1.1  2000/02/02 21:38:56  klaus
# Preparing WWWdb for 0.0.6
#
#
# -----------------------------------------------------------------------------

use DBI;
use CGI;
use Getopt::Std;
use File::Basename;
use FileHandle;
use Carp;
use Data::Dumper;

use WWWdb::DbSession;

$Data::Dumper::Indent = 1;         # mild pretty print

use vars qw($opt_b $opt_c $opt_d $opt_e $opt_f $opt_i $opt_s $opt_t
            $opt_D $opt_F $opt_U $opt_P $opt_H $opt_I $opt_W $opt_V);
use strict;

$main::cVersionMGL = '$Date: 2002/03/18 14:36:27 $$Revision: 1.10 $ ';
$main::cProgramMGL = basename($0);

my $oDbSessionGL;


getopts("b:cdefh:ist:D:F:H:IP:U:VW:") || Usage();

# Print version
if ($opt_V)
{
    print "$0: $main::cVersionMGL\n";
    exit;
}

Usage() unless @ARGV;

Main();

# --- show the usage of the script  ---------------------------------------------
sub Usage()
{
    print <<EOF;
    Call:
        $main::cProgramMGL -D <database> -U <user>
            [-P <password>] [-H <host>]
            [-b Oracle|Pg|mysql|InterBase|csv|ODBC]
            [-c][-e [-F <Field-List>] [-W <WHERE-clause>]][-i]
            [-s] [-dV]
            <xxx.dbdef> ... <xxx> ... <xxx.csv> ...

    Switches:
        -b <database-driver>
               at the moment, the following drivers are implemented:
               Pg        Postgres
               mysql     MySql
               Oracle    Oracle
               InterBase InterBase
               ODBC     Generic ODBC-driver

        -D     database
        -U     user
        -P     password
        -H     host (localhost)
        -F     list of fields to export, separated by comma
        -W     where-clause for export
        -s     silent-mode
        -d     Debugging-mode
        -I     Re-create indices (implies -c)
        -V     show version
        -f     force operation
        -c     create tables (*.dbdef)
        -e     export tables (*)
        -i     import tables (*.csv)

    Description:
        $main::cProgramMGL is used to create, import and export
        database-tables.

EOF
    exit 0;
}


# --- Connect to the database  ---------------------------------
sub ConnectDB (;$)
{
   my ($cConnectStringPI,
       $cDbDriverPI,
       $iDbiTraceLevelPI)  = @_;

   my $cDbSid  = $opt_D;
   my $cDbUser = $opt_U;
   my $cDbPass = $opt_P;
   my $cDbHost = $opt_H;

   if (!$opt_D || !$opt_U)
   {
      print STDERR ("Database (-D) and username (-U) are needed\n");
      MyExit(-1);
   }

   if (!$opt_b)
   {
      print STDERR ("Database-type (-b) is needed\n");
      MyExit(-1);
   }

   DBI->trace($iDbiTraceLevelPI) if(defined $iDbiTraceLevelPI);

   $oDbSessionGL = DbSession->new($opt_b,
                                  $cDbSid,
                                  $cDbUser,
                                  $cDbPass,
                                  # patch from Pirmin Kalberer <pka@sourcepole.ch>
                                  DbHost   => $cDbHost,
                                  DebugLvl => $opt_d);
}

# --- Import csv-file into the database ----------------------------
sub ImportInDB($)
{
    my $cImportFileNamePI = shift;

    my $cFieldList;
    my $cSqlStmt;
    my $cSqlStmtTmpl;
    my $cTableName;
    my $hStmt;
    my $iRecordsImported;
    my $iRecordsWithError;
    my $iResult,

    open IMPORT, "<$cImportFileNamePI" or
        die "Import-file ´$cImportFileNamePI´ could not be opened!\n";

    printf STDERR "Importing %s ...\n", $cImportFileNamePI
        unless $opt_s;

    $cTableName = ($opt_t?
                   $opt_t:
                   basename($cImportFileNamePI, ".csv"));

    if (!$cTableName) {
       print STDERR ("Table-name is missing!\n" .
                     "Declare via import-file <table>.csv or\n" .
                     "use switch -t <table>.\n");
       MyExit(-1);
    }

    chomp($cFieldList = <IMPORT>);

    print STDERR "Field-list: $cFieldList\n"
        if $opt_d;

    $iRecordsImported = 0;
    $iRecordsWithError = 0;

    # read csv-file
    while (<IMPORT>)
    {

       chomp;

       # s/\"/\'/g;

       # put all \\n back to \n
       s/\\\\n/\n/g;

       # Handle empty values
       s/,,/,NULL,/g;
       s/^,/NULL,/;
       s/,$/,NULL/;

       s/@!<DATE>(.*?)@!<\/DATE>/$oDbSessionGL->DateToDb($1)/eg;

       unless ($opt_s)
       {
          print STDERR ($iRecordsImported % 10?
                        ".":
                        "+");
       }

       print STDERR"$_\n"
           if $opt_d;

       $cSqlStmt =
           "INSERT INTO $cTableName ($cFieldList) VALUES ($_)";
       $iResult = &ExecSQL($cSqlStmt, 0);

       if($iResult)
       {
          print STDERR "Press Enter:";
          <STDIN>;
          $iRecordsWithError ++;
       }
       else
       {
          $iRecordsImported ++;
       }

    } # while (<IMPORT>)

    close IMPORT;

    printf STDERR
        ("\nOK\n" .
         "%d records were successfully imported\n" .
         "%d errors were detected\n\n",
         $iRecordsImported,
         $iRecordsWithError) unless $opt_s;

} # sub ImportInDB



# --- Export a database-table into an csv-ascii-file ----------------------
sub ExportFromDB ($)
{
   my $cTablePI = shift;

   my $cLine;
   my $oStmtHdl;
   my @lRecord;
   my $iCnt;
   my $pTypeInfo;
   my $bQuote;
   my $oDbHandle;
   my $cSqlCommand = sprintf("SELECT %s FROM %s %s",
                             ($opt_F?
                              $opt_F:
                              "*"),
                             $cTablePI,
                             ($opt_W?
                              "WHERE " . $opt_W :
                              ""));

   $oDbHandle = $oDbSessionGL->getDbHandle();

   $oStmtHdl = $oDbHandle->prepare($cSqlCommand)
           or die "Cant prepare $cSqlCommand\n";
   $oStmtHdl->execute()
       or die "Cant execute $cSqlCommand\n";

   $iCnt = 0;

   printf "Exporting %s ...\n", $cTablePI
        unless $opt_s;

   open EXPORT, ">${cTablePI}.csv" or
        die "Export-file ´${cTablePI}.csv´ could not be opened!\n";

   printf STDERR "$cSqlCommand\n"
       if $opt_d;

   # append field-names
   foreach (@{$oStmtHdl->{NAME}})
   {
      $cLine .=  $_ . ",";
   }

   # remove last comma
   $cLine =~ s/,$//;

   print EXPORT "$cLine\n";


   while(@lRecord = $oStmtHdl->fetchrow_array)
   {
      my $iInd;

      $cLine = "";

      # append field-names
      for ($iInd = 0; $iInd < @lRecord; $iInd ++)
      {
         my $rhTypeInfo;
         my $cValue        = $lRecord[$iInd];
         my $iDataType     = $oStmtHdl->{TYPE}->[$iInd];
         my $iOrigDataType = $oStmtHdl->{TYPE}->[$iInd];

         # some databases (eg. Postgres) need re-mapping to DBI-defaults
		 # in this case the DBI-constants like DBI::SQL_TIMESTAMP can be used
		 # But attention: for type_info the original must be used
         $iDataType = &DbSession::MapDatatype($iDataType)
			 if ($iDataType > 20);

         # datetime
         if ($iDataType == DBI::SQL_TIMESTAMP)
         {
            if ($cValue eq "")
            {
               $cValue = undef;
            }
            else
            {
               $cValue = ("@!<DATE>" .
                          $oDbSessionGL->DateFromDb($cValue).
                          "@!</DATE>");
            }
         }

         # sorry, but DBD::mysql does not support type-info
         if (lc("$opt_b") eq "mysql")
         {
            $bQuote = "Yes";
         }
         else
         {
            $pTypeInfo = $oDbHandle->type_info($iOrigDataType);
            $bQuote = $pTypeInfo->{LITERAL_PREFIX};
         }

         $cValue =~ s/\n/\\n/g;

         if (! defined $cValue)
         {
            $cValue = "NULL";
            $bQuote = "";
         }

         if ($bQuote)
         {

            # sorry, but DBD::mysql does not support the second form
            if (lc("$opt_b") eq "mysql")
            {
               $cLine .= $oDbHandle->quote($cValue);
            }
            else
            {
               $cLine .= $oDbHandle->quote($cValue, $iOrigDataType);
            }
         }
         else
         {
            $cLine .= $cValue;
         }

         $cLine .= ","
             if $iInd < (@lRecord - 1);

      }

      $iCnt ++;

      print EXPORT "$cLine\n";
   }

   $oStmtHdl->finish();

   close EXPORT;

   printf STDERR ("%d records were exported\n", $iCnt)
       unless $opt_s;

   MyExit(0);
}



# --- Create a Table in the database  -----------------------------------------
sub CreateTableDB (@)
{

   my %hFileDescPI = @_;

   my $cIndex;
   my $cSqlIndexStmt;
   my $bAddComma;
   my $cDataType;
   my $cExecCreate;
   my $cField;
   my $cLabel;
   my $cSqlDropStmt;
   my $cSqlCreateStmt;
   my $cSqlRevokeStmt;
   my $cSqlGrantStmt;
   my $hStmt;
   my $iLen;
   my $iResult;
   my %hSqlDataTypes;
   my @liLen;

   $cSqlCreateStmt = sprintf("CREATE TABLE %s (\n",
                             $hFileDescPI{"TABLE"});

   $bAddComma = 0;

   foreach $cField (grep /FIELD_(\d+)/, sort keys %hFileDescPI)
   {

      if($bAddComma) {
     $cSqlCreateStmt .= ",\n";
      }
      else
      {
     $bAddComma = 1;
      }

      if($hFileDescPI{$cField}{"TYPE"} =~ /(\w+)\s*\((\d+)\)/)
      {
         $iLen = $2;
         $cDataType = $1;
      }
      else
      {
         $iLen = 0;
         $cDataType = $hFileDescPI{$cField}{"TYPE"};
      }

      if (lc("$opt_b") eq "pg") {

         %hSqlDataTypes =
             ("varchar"         => "varchar",
              "char"            => "char",
              "text"            => "text",
              "int"             => "int4",
              "integer"         => "int4",
              "float"           => "float8",
              "date"            => "datetime",
              "tlob"            => "text",
              "_max_varchar"    => "4048",
              "_has_grant"      => "Y",
              "_has_revoke"     => "Y",
              "_has_index"      => "Y",
              "_types_to_quote" => ",varchar,char,text,datetime,",
              );
      }
      # this is for testing purposes only, because adabas does seem to
      # support only records with a max size of 4096
      elsif (lc("$opt_b") eq "adabas") {
         %hSqlDataTypes =
             ("varchar"      => "varchar",
              "char"         => "char",
              "text"         => "varchar(1500)",
              "int"          => "integer",
              "integer"      => "integer",
              "float"        => "float",
              "date"         => "timestamp",
              "tlob"         => "varchar",
              "_max_varchar" => "1024",
              "_has_grant"   => "Y",
              "_has_revoke"  => "Y",
              "_has_index"   => "Y",
              "_types_to_quote" => ",varchar,char,long char,timestamp,",
              );

      }
      elsif (lc("$opt_b") eq "csv") {
         %hSqlDataTypes =
             ("varchar"      => "char",
              "char"         => "char",
              "text"         => "char",
              "int"          => "char(32)",
              "integer"      => "char(32)",
              "float"        => "char(32)",
              "date"         => "char(32)",
              "tlob"         => "text",
              "_max_varchar" => "1000000",
              "_has_grant"   => "N",
              "_has_revoke"  => "N",
              "_has_index"   => "N",
              "_types_to_quote" => ",char,",
              );

      }
      elsif (lc("$opt_b") eq "mysql") {
         %hSqlDataTypes =
             ("varchar"      => "varchar",
              "char"         => "char",
              "text"         => "text",
              "int"          => "integer",
              "integer"      => "integer",
              "float"        => "float",
              "date"         => "datetime",
              "tlob"         => "text",
              "_max_varchar" => "255",
              "_has_grant"   => "Y",
              "_has_revoke"  => "N",
              "_has_index"   => "Y",
              "_types_to_quote" => ",varchar,char,text,datetime,",
              );

      }
      elsif (lc("$opt_b") eq "oracle") {
         %hSqlDataTypes =
             ("varchar"      => "varchar2",
              "char"         => "char",
              "text"         => "varchar2",
              "int"          => "integer",
              "integer"      => "integer",
              "float"        => "float",
              "date"         => "date",
              "tlob"         => "long",
              "_max_varchar" => "2048",
              "_has_grant"   => "Y",
              "_has_revoke"  => "Y",
              "_has_index"   => "Y",
              "_types_to_quote" => ",varchar2,char,text,date,",
              );
      }
      elsif (lc("$opt_b") eq "interbase") {
         %hSqlDataTypes =
             ("varchar"      => "varchar",
              "char"         => "char",
              "text"         => "varchar",
              "int"          => "integer",
              "integer"      => "integer",
              "float"        => "float",

              # Please note, that timestamp can not be used currently, because
              # a SELECT returns a format, that is not understood by
              # Interbase SQL
              "date"         => "date",
              # "tlob"         => "blob",
              "tlob"         => "varchar(16384)",
              "_max_varchar" => "32000",
              "_has_grant"   => "Y",
              "_has_revoke"  => "Y",
              "_has_index"   => "Y",
              "_types_to_quote" => ",varchar,char,blob,timestamp,",
              );
      }
      elsif (lc("$opt_b") eq "odbc") {
         %hSqlDataTypes =
             ("varchar"      => "varchar",
              "char"         => "char",
              "text"         => "longvarchar",
              "int"          => "integer",
              "integer"      => "integer",
              "float"        => "float",
              "date"         => "timestamp",
              "tlob"         => "longvarchar",
              "_max_varchar" => "254",
              "_has_grant"   => "Y",
              "_has_revoke"  => "Y",
              "_has_index"   => "Y",
              "_types_to_quote" => ",varchar,char,longvarchar,date,",
              );
      }
      elsif (lc("$opt_b") eq "odbc_pg") {
         %hSqlDataTypes =
             ("varchar"      => "varchar",
              "char"         => "char",
              "text"         => "text",
              "int"          => "integer",
              "integer"      => "integer",
              "float"        => "float",
              "date"         => "timestamp",
              "tlob"         => "text",
              "_max_varchar" => "254",
              "_has_grant"   => "Y",
              "_has_revoke"  => "Y",
              "_has_index"   => "Y",
              "_types_to_quote" => ",varchar,char,longvarchar,date,",
              );
      }
      else
      {
         %hSqlDataTypes =
             ("varchar"      => "char",
              "char"         => "char",
              "int"          => "char(32)",
              "integer"      => "char(32)",
              "float"        => "char(32)",
              "date"         => "char(32)",
              "tlob"         => "text",
              "_max_varchar" => "255",
              "_has_grant"   => "N",
              "_has_revoke"  => "N",
              "_has_index"   => "N",
              "_types_to_quote" => ",char,",
              );

      }

      if (($cDataType eq "varchar") &&
          ($iLen > $hSqlDataTypes{"_max_varchar"}))
      {
         my $iOldLen = $iLen;

         # if wanted length is greater than the half of _max_varchar
         # the field gets cutted!
         if($iLen < $hSqlDataTypes{"_max_varchar"} * 2)
         {
            $iLen = $hSqlDataTypes{"_max_varchar"};
         }
         else
         {
            $iLen = 0;
            $cDataType = "tlob";
         }

         printf STDERR
             ("Warning: length %d of field %s exceeds length-limit of %d!\n" .
              "         Convert it to type %s.\n",
              $iOldLen,
              $hFileDescPI{$cField}{"NAME"},
              $hSqlDataTypes{"_max_varchar"},
              uc($hSqlDataTypes{$cDataType}) .
              ($iLen? "($iLen)": ""));
      }

      if(defined($hSqlDataTypes{$cDataType}))
      {
         $cDataType = $hSqlDataTypes{$cDataType};
      }
      else
      {
         die sprintf "Datatype '%s' is unknown!\n", $cDataType;
      }

      # append field
      $cSqlCreateStmt .= sprintf("    %-20s %-20s %s",
                 $hFileDescPI{$cField}{"NAME"},
                 uc(($iLen ?
                                     "$cDataType($iLen)":
                                     "$cDataType")),
                                 $hFileDescPI{$cField}{"EXTRA"});

   } # foreach $cField ...

   $cSqlCreateStmt .= "\n)";

   if ($opt_f && !$opt_I)
   {
      &ExecSQL(sprintf("DROP TABLE %s\n",
                       $hFileDescPI{"TABLE"}), 0);
   }

   unless($opt_I)
   {

      printf STDERR
          ("Create table %s ...\n",
           $hFileDescPI{"TABLE"})
              unless $opt_s;

      &ExecSQL($cSqlCreateStmt, 1);
   }

   if(($hSqlDataTypes{"_has_revoke"} eq "Y") &&
      !$opt_I)
   {
      $cSqlRevokeStmt = sprintf("REVOKE ALL ON %s FROM PUBLIC\n",
                                $hFileDescPI{"TABLE"});

      &ExecSQL($cSqlRevokeStmt, 1);
   }

   if(($hSqlDataTypes{"_has_grant"} eq "Y") &&
      !$opt_I)
   {
      $cSqlGrantStmt = sprintf("GRANT ALL ON %s TO PUBLIC\n",
                               $hFileDescPI{"TABLE"});

      &ExecSQL($cSqlGrantStmt, 1);
   }

   if($hSqlDataTypes{"_has_index"} eq "Y")
   {
     foreach $cIndex (grep /INDEX_(\d+)/, sort keys %hFileDescPI)
     {
        printf STDERR
            ("Create index %s_%s ...\n",
             $hFileDescPI{"TABLE"},
             $hFileDescPI{$cIndex}{"NAME"})
                unless $opt_s;

        if ($opt_I)
        {
           $cSqlIndexStmt =
               sprintf("DROP INDEX %s_%s\n",
                       $hFileDescPI{"TABLE"},
                       $hFileDescPI{$cIndex}{"NAME"});
           &ExecSQL($cSqlIndexStmt);
        }

        $cSqlIndexStmt =
            sprintf("CREATE %sINDEX %s_%s ON %s\n\t(%s)\n",
                    (uc($hFileDescPI{$cIndex}{"TYPE"}) eq "PRIMARY"?
                     "UNIQUE ":
                     ""),
                    $hFileDescPI{"TABLE"},
                    $hFileDescPI{$cIndex}{"NAME"},
                    $hFileDescPI{"TABLE"},
                    $hFileDescPI{$cIndex}{"FIELDS"});

        &ExecSQL($cSqlIndexStmt, 1);

      }
   }
}

sub ExecSQL($$)
{
   my $cSqlStmtPI      = shift;
   my $bAbortOnErrorPI = shift;

   my $cResult;

   print STDERR "$cSqlStmtPI\n"
       if $opt_d;

   if ($bAbortOnErrorPI)
   {
      $oDbSessionGL->SqlDo($cSqlStmtPI);
   }
   else
   {
      unless (eval {
         $oDbSessionGL->SqlDo($cSqlStmtPI);
      })
      {
         warn @_;
          print STDERR "Press Enter:";
         <STDIN>;
      }
   }


   return $cResult;
}



sub ReadDbdefFile($)
{
   my $cFileNamePI = shift;
   my %hFileDesc = ();
   my $iFieldSeq;
   my $iIndexSeq;

   open DBDESC, "<$cFileNamePI" or
       die "Can't open Db-Description-File: ´$cFileNamePI´!\n";

   $iFieldSeq = 0;
   $iIndexSeq   = 0;
   while (<DBDESC>) {

      chomp;

      # comment
      if (/^\#.*/)
      {
         next;
      }
      # Table-Name
      elsif (/^table\s+(\w+)/i)
      {
         $hFileDesc{"TABLE"} = $1;
      }
      # Fields
      elsif (/^field\s+(\w+)\s+([\w()]+)(\s+(.+))?/i)
      {
         my $cKey = sprintf ("FIELD_%03d", $iFieldSeq);

         $iFieldSeq ++;
         $hFileDesc{$cKey}{"NAME"}  = $1;
         $hFileDesc{$cKey}{"TYPE"}  = $2;
         $hFileDesc{$cKey}{"EXTRA"} = $4;
      }
      elsif (/^index\s+(\w+)\s+(\w+)\s+(.+)/i)
      {
         my $cKey = sprintf ("INDEX_%03d", $iIndexSeq);

         $iIndexSeq ++;
         $hFileDesc{$cKey}{"TYPE"}  = $1;
         $hFileDesc{$cKey}{"NAME"}  = $2;
         $hFileDesc{$cKey}{"FIELDS"} = $3;
      }
   }

   close DBDESC;

   print STDERR Dumper(%hFileDesc)
       if $opt_d;

   return %hFileDesc;

}



# --- Hauptprozedur -----------------------------------------------------------
sub Main () {

    my $cFileName;

    # -I implies -c
    $opt_c = 1
        if($opt_I);



    STDOUT->autoflush(1);

    &ConnectDB($opt_b, 0);

    if ($opt_c)
    {
       # create all tables defined in *.dbdef-files
       foreach $cFileName (grep /^.*\.dbdef$/, @ARGV)
       {
          &CreateTableDB(&ReadDbdefFile($cFileName));
       }
    }

    if ($opt_i)
    {
       # import all tables defined in *.csv-files
       foreach $cFileName (grep /^.*\.csv$/, @ARGV)
       {
          &ImportInDB($cFileName);
       }
    }

    if ($opt_e)
    {
       # export all given tables
       foreach $cFileName (grep /^\w+$/, @ARGV)
       {
          &ExportFromDB($cFileName);
       }
    }

    MyExit(0);
} # sub Main



# --- Das Programm definiert verlassen ----------------------------------------
sub MyExit($)
{
   my $iResultPI = shift;

   # Angefangene Zeilen abschließen
   print STDERR "\n"
       unless $opt_s;

   # Datenbank-Verbindung abbauen
   exit ($iResultPI);

}
