mvhub-dev team mailing list archive

[Dan MacNeil <dan@xxxxxxxxxx>]

Dan MacNeil has proposed merging lp:~omacneil/mvhub/remove_warnings_from_test_data into lp:mvhub.

Requested reviews:
  MVHub devs with commit rights (mvhub-commit)


The test data is all kinds of not UTF8

If you don't specify an ENCODING when you create a database, you get UTF8 in postgresql 8.3

So if the database is set to UTF8 and the data to load has client_encoding SQL_ASCII  and data in iso-8859-01, ---lots of errors, some of them fatal

So until we clean up the data we create the databases as SQL_ASCII
-- 
https://code.launchpad.net/~omacneil/mvhub/remove_warnings_from_test_data/+merge/24534
Your team MVHub Developers is subscribed to branch lp:mvhub.

=== modified file 'app-mvhub/doc/checklists/install_mvhub.txt'
--- app-mvhub/doc/checklists/install_mvhub.txt	2010-04-17 00:06:46 +0000
+++ app-mvhub/doc/checklists/install_mvhub.txt	2010-04-30 21:37:23 +0000
@@ -167,6 +167,10 @@
      
 ___ create database(s) and database user
 
+  # Make template1 connectable to all because our programs require a database
+  # to connect to when dropping and adding user databases.
+  	sudo su postgres -c "psql -c 'GRANT CONNECT ON DATABASE template1 TO PUBLIC;'"
+  
   # add database user 
   # note the password you assign to database user
      sudo su postgres -c "createuser -A -D -E -P -R $USER"

=== modified file 'app-mvhub/project-tools/bin/mv_setup'
--- app-mvhub/project-tools/bin/mv_setup	2010-04-23 20:12:38 +0000
+++ app-mvhub/project-tools/bin/mv_setup	2010-04-30 21:37:23 +0000
@@ -17,8 +17,7 @@
 use IO::Prompt;
 use Config::Simple;
 use MVHub::Utils::ConfigSimple;
-use MVHub::Utils::DB qw/get_dbh/;
-
+use MVHub::Utils::DB qw/get_data_source /;
 my @WEBSITE_CODES = qw/ mvh nsp /;
 
 {    # main
@@ -39,12 +38,12 @@
     make_databases_for( $USERNAME, @WEBSITE_CODES );
 
     make_config_files_for( $USERNAME, $cfg, @WEBSITE_CODES );
-    
-    if (defined $ENV{SUDO_USER}){
-       foreach my $prefix (@WEBSITE_CODES){
-          add_dns($USERNAME,$prefix);
-          add_apache_config($USERNAME,$prefix);
-       }
+
+    if ( defined $ENV{SUDO_USER} ) {
+        foreach my $prefix (@WEBSITE_CODES) {
+            add_dns( $USERNAME, $prefix );
+            add_apache_config( $USERNAME, $prefix );
+        }
     }
 }
 
@@ -52,13 +51,13 @@
     my $username = shift or croak 'missing paramater: $username';
     my $prefix   = shift or croak 'missing parameter: $prefix';
 
-    my $host="$prefix.$username";
-    my $FILE='/etc/bind/db.testing123.net';
-        
-    my $line = sprintf( "%-15s IN      A       129.63.96.180\n", $host);
-    if (possible_dup_line($host,$FILE)){
-       warn "$host found in $FILE, skipping append\n";
-       return;
+    my $host = "$prefix.$username";
+    my $FILE = '/etc/bind/db.testing123.net';
+
+    my $line = sprintf( "%-15s IN      A       129.63.96.180\n", $host );
+    if ( possible_dup_line( $host, $FILE ) ) {
+        warn "$host found in $FILE, skipping append\n";
+        return;
     }
 
     append_to_file( $FILE, $line );
@@ -68,15 +67,14 @@
     my $username = shift or croak 'missing paramater: $username';
     my $prefix   = shift or croak 'missing parameter: $prefix';
 
-    my $FILE     = '/etc/apache2/sites-available/mvhub_developer.data';
-    my $host     = "$prefix.$username.testing123.net";
-    
-    my $line = sprintf( "Use MVhub %-15s $prefix $host\n", $username 
-    );
-
-    if (possible_dup_line($host,$FILE)){
-       warn "$host found in $FILE, skipping append\n";
-       return;
+    my $FILE = '/etc/apache2/sites-available/mvhub_developer.data';
+    my $host = "$prefix.$username.testing123.net";
+
+    my $line = sprintf( "Use MVhub %-15s $prefix $host\n", $username );
+
+    if ( possible_dup_line( $host, $FILE ) ) {
+        warn "$host found in $FILE, skipping append\n";
+        return;
     }
 
     append_to_file( $FILE, $line );
@@ -227,6 +225,30 @@
       or die "$cmd failed ABORTING\n";
 }
 
+sub get_dbh {
+    my $config_file = shift or croak "missing param: config_file";
+     croak "couldn't read config_file: '$config_file'"
+      if not -r $config_file;
+
+    my $data_source = MVHub::Utils::DB::get_data_source($config_file);
+
+    my $cfg               = new Config::Simple($config_file);
+    my $database_username = $cfg->param('DATABASE.database_user');
+    my $auth              = $cfg->param('DATABASE.database_magic_word');
+
+    my %attr = (
+        'PrintError'         => 1,
+        'RaiseError'         => 1,
+        'AutoCommit'         => 1,
+        'ShowErrorStatement' => 1,
+    );
+
+    my $dbh = DBI->connect('dbi:Pg:dbname=template1',$database_username, $auth, \%attr )
+      or die "couldn't connect to database";
+    return $dbh;
+
+}
+
 sub get_dirs_with {
     my $base_dir = shift or croak 'missing param: $base_dir';
     my $username = shift or croak 'missing param: $username';
@@ -461,7 +483,7 @@
         # kludge to avoid 'db being currently being accessed' errs
         sleep 1;
 
-        $sql = qq{CREATE DATABASE "$username.${suffix}" OWNER $username};
+        $sql = qq{CREATE DATABASE "$username.${suffix}" OWNER $username ENCODING='SQL_ASCII' };
         $dbh->do($sql);
 
         load_test_data_for( $dbh, $username, $suffix );
@@ -492,7 +514,7 @@
 
         print
           "(re)making databases for $username this may take 50 seconds....\n";
-        my $DBH = MVHub::Utils::DB::get_dbh( $ENV{MV_CONFIG_FILE} );
+        my $DBH = get_dbh( $ENV{MV_CONFIG_FILE} );
 
         print "...done\n"
           if make_dbs_for( $DBH, $username, @website_codes );
@@ -573,15 +595,15 @@
 
 }
 
-sub possible_dup_line{
-    my $host =shift or croak 'missing parameter: $host';
-    my $filename =shift or croak 'missing parameter: $filename';
-
-    open my $in_fh, $filename 
-    	or die "failed to open $filename error:$!";
-    
-    my @contents=<$in_fh>;
-    my $regex=qr/$host/;
-    return scalar grep {$_=~$regex} @contents;
+sub possible_dup_line {
+    my $host     = shift or croak 'missing parameter: $host';
+    my $filename = shift or croak 'missing parameter: $filename';
+
+    open my $in_fh, $filename
+      or die "failed to open $filename error:$!";
+
+    my @contents = <$in_fh>;
+    my $regex    = qr/$host/;
+    return scalar grep { $_ =~ $regex } @contents;
 }
 

=== modified file 'app-mvhub/project-tools/test_mvh_db.sql'
--- app-mvhub/project-tools/test_mvh_db.sql	2010-03-28 13:07:47 +0000
+++ app-mvhub/project-tools/test_mvh_db.sql	2010-04-30 21:37:23 +0000
@@ -10,7 +10,6 @@
 -- Name: SCHEMA public; Type: COMMENT; Schema: -; Owner: postgres
 --
 
-COMMENT ON SCHEMA public IS 'Standard public schema';
 
 
 SET search_path = public, pg_catalog;
@@ -10001,12 +10000,6 @@
 -- Name: public; Type: ACL; Schema: -; Owner: postgres
 --
 
-REVOKE ALL ON SCHEMA public FROM PUBLIC;
-REVOKE ALL ON SCHEMA public FROM postgres;
-GRANT ALL ON SCHEMA public TO postgres;
-GRANT ALL ON SCHEMA public TO PUBLIC;
-
-
 --
 -- PostgreSQL database dump complete
 --