Index: admin/.htaccess
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/connect.php,v
retrieving revision 1.43
diff -c -r1.43 connect.php
*** admin/connect.php	9 Dec 2005 16:11:23 -0000	1.43
--- admin/connect.php	1 Aug 2006 12:02:25 -0000
***************
*** 31,39 ****
  
  # identify pages that can be run on commandline
  $commandline_pages = array("send","processqueue","processbounces","getrss");
  
! if (isset($message_envelope))
!   $envelope = "-f$message_envelope";
  $database_connection = Sql_Connect($database_host,$database_user,$database_password,$database_name);
  
  if (!empty($GLOBALS["SessionTableName"])) {
--- 31,39 ----
  
  # identify pages that can be run on commandline
  $commandline_pages = array("send","processqueue","processbounces","getrss");
+ $commandline_pages = array("sequence", "send","processqueue","processbounces","getrss");
  
! $envelope = isset($message_envelope) ? "-f$message_envelope" : "";
  $database_connection = Sql_Connect($database_host,$database_user,$database_password,$database_name);
  
  if (!empty($GLOBALS["SessionTableName"])) {
***************
*** 56,61 ****
--- 56,62 ----
    "messagedata" => $table_prefix. "messagedata",
    "listmessage" => $table_prefix . "listmessage",
    "usermessage" => $table_prefix . "usermessage",
+   "usermessage_seq" => $table_prefix . "usermessage_seq",
    "attribute" => $usertable_prefix . "attribute",
    "user_attribute" => $usertable_prefix . "user_attribute",
    "sendprocess" => $table_prefix . "sendprocess",
Index: admin/index.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/index.php,v
retrieving revision 1.26
diff -c -r1.26 index.php
*** admin/index.php	2 Dec 2005 00:57:15 -0000	1.26
--- admin/index.php	1 Aug 2006 12:02:27 -0000
***************
*** 428,431 ****
    }
    return $res;
  }
! 
--- 428,431 ----
    }
    return $res;
  }
! ?>
Index: admin/members.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/members.php,v
retrieving revision 1.8
diff -c -r1.8 members.php
*** admin/members.php	9 Sep 2005 16:10:18 -0000	1.8
--- admin/members.php	1 Aug 2006 12:02:28 -0000
***************
*** 159,164 ****
--- 159,165 ----
         values("%s",now(),1,%d,"%s")',
        $tables["user"],$email,$htmlemail,getUniqid()));
      $userid = Sql_insert_id();
+     trigger_user_confirmed($userid);
      $query = "insert into $tables[listuser] (userid,listid,entered)
   values($userid,$id,now())";
      $result = Sql_query($query);
***************
*** 178,186 ****
          $tables["user_attribute"],$row["id"],$userid,$value));
      }
    } else {
!     $query = "replace into $tables[listuser] (userid,listid,entered)
!  values({$_REQUEST["doadd"]},$id,now())";
      $result = Sql_query($query);
    }
    echo "<br /><font color=red size=+2>".$GLOBALS['I18N']->get("User added")."</font><br />";
  }
--- 179,188 ----
          $tables["user_attribute"],$row["id"],$userid,$value));
      }
    } else {
!     $listid = $_REQUEST['doadd'];
!     $query = "replace into {$tables[listuser]} (userid,listid,entered) values({$listid,$id,now())";
      $result = Sql_query($query);
+     trigger_listuser_added($listid, $id); 
    }
    echo "<br /><font color=red size=+2>".$GLOBALS['I18N']->get("User added")."</font><br />";
  }
Index: admin/processqueue.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/processqueue.php,v
retrieving revision 1.19
diff -c -r1.19 processqueue.php
*** admin/processqueue.php	2 Dec 2005 00:57:15 -0000	1.19
--- admin/processqueue.php	1 Aug 2006 12:02:31 -0000
***************
*** 306,763 ****
    output($GLOBALS['I18N']->get('Skipped in last run').": $lastskipped");
  }
  
! $script_stage = 1; # we are active
! $notsent = $sent = $invalid = $unconfirmed = $cannotsend = 0;
  
! $messages = Sql_query("select id,userselection,rsstemplate,subject from ".$tables["message"]." where status != \"draft\" and status != \"sent\" and status != \"prepared\" and status != \"suspended\" and embargo < now() order by entered");
! $num_messages = Sql_affected_rows();
! if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
! 
! if ($num_messages) {
!   output($GLOBALS['I18N']->get('Processing has started,').' '.$num_messages.' '.$GLOBALS['I18N']->get('message(s) to process.'));
!   if (!$GLOBALS["commandline"]) {
!     if (!$safemode) {
!       if (!$num_per_batch) {
!         output($GLOBALS['I18N']->get('It is safe to click your stop button now, report will be sent by email to').' '.getConfig("report_address"));
        } else {
!         output($GLOBALS['I18N']->get('Please leave this window open. You have batch processing enabled, so it will reload several times to send the messages. Reports will be sent by email to').' '.getConfig("report_address"));
        }
-     } else {
-       output($GLOBALS['I18N']->get('Your webserver is running in safe_mode. Please keep this window open. It may reload several times to make sure all messages are sent.').' '.$GLOBALS['I18N']->get('Reports will be sent by email to').' '.getConfig("report_address"));
      }
    }
- }
  
! Sql_query("SET SQL_BIG_TABLES=1");
! $script_stage = 2; # we know the messages to process
! include_once "footer.inc";
! if (!$num_per_batch) {
!   $num_per_batch = 1000000;
! }
! 
! while ($message = Sql_fetch_array($messages)) {
!   $failed_sent = 0;
!   $throttlecount = 0;
! 
!   $messageid = $message["id"];
!   $userselection = $message["userselection"];
!   $rssmessage = $message["rsstemplate"];
! 
!   $msgdata = loadMessageData($messageid);
!   if (!empty($msgdata['notify_start']) && !isset($msgdata['start_notified'])) {
!     $notifications = explode(',',$msgdata['notify_start']);
!     foreach ($notifications as $notification) {
!       sendMail($notification,$GLOBALS['I18N']->get('Message Sending has started'),
!         sprintf($GLOBALS['I18N']->get('phplist has started sending the message with subject %s'),$message['subject']."\n".
!         sprintf($GLOBALS['I18N']->get('to view the progress of this message, go to %s'),getConfig('website').$GLOBALS['adminpages'].'/?page=messages&type=sent')));
      }
-     Sql_Query(sprintf('insert ignore into %s (name,id,data) values("start_notified",%d,now())',
-       $GLOBALS['tables']['messagedata'],$messageid));
-   }
  
!   output($GLOBALS['I18N']->get('Processing message').' '. $messageid);
!   if (ENABLE_RSS && $message["rsstemplate"]) {
!     $processrss = 1;
!     output($GLOBALS['I18N']->get('Message').' '. $messageid.' '.$GLOBALS['I18N']->get('is an RSS feed for').' '. $GLOBALS['I18N']->get($rssmessage));
!   } else {
!     $processrss = 0;
!   }
! 
!   flush();
!   keepLock($send_process_id);
!   $status = Sql_query('update '.$tables["message"].' set status = "inprocess" where id = '.$messageid);
!   $sendstart = Sql_query('update '.$tables["message"].' set sendstart = now() where sendstart is NULL and id = '.$messageid);
!   output($GLOBALS['I18N']->get('Looking for users'));
!   if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
! 
!   # make selection on attribute, users who at least apply to the attributes
!   # lots of ppl seem to use it as a normal mailinglist system, and do not use attributes.
!   # Check this and take anyone in that case.
!   $numattr = Sql_Fetch_Row_Query("select count(*) from ".$tables["attribute"]);
! 
!   if ($userselection && $numattr[0]) {
!     $res = Sql_query($userselection);
!     $num_users = Sql_Affected_rows($res);
!     output($num_users.' '.$GLOBALS['I18N']->get('users apply for attributes, now checking lists'));
!     $user_list = "";
!     while ($row = Sql_Fetch_row($res)) {
!       $user_list .= $row[0] . ",";
!     }
!     $user_list = substr($user_list,0,-1);
!     if ($user_list)
!       $user_attribute_query = " and listuser.userid in ($user_list)";
!     else {
!       output($GLOBALS['I18N']->get('No users apply for attributes'));
!       $status = Sql_query("update {$tables["message"]} set status = \"sent\",sent = now() where id = \"$messageid\"");
!       finish("info","Message $messageid: \nNo users apply for attributes, ie nothing to do");
!       $script_stage = 6;
!       # we should actually continue with the next message
!       return;
      }
-   }
-   if ($script_stage < 3)
-     $script_stage = 3; # we know the users by attribute
  
!   # when using commandline we need to exclude users who have already received
!   # the email
!   # we don't do this otherwise because it slows down the process, possibly
!   # causing us to not find anything at all
!   $exclusion = "";
!   $doneusers = array();
!   $skipusers = array();
!   if (VERBOSE) {
!     output($GLOBALS['I18N']->get('looking for users who can be excluded from this mailing'));
!   }
!   $req = Sql_Query("select userid from {$tables["usermessage"]} where messageid = $messageid");
!   $skipped = Sql_Affected_Rows();
!   while ($row = Sql_Fetch_Row($req)) {
!     $alive = checkLock($send_process_id);
!     if ($alive)
!       keepLock($send_process_id);
!     else
!       ProcessError($GLOBALS['I18N']->get('Process Killed by other process'));
!     array_push($doneusers,$row[0]);
!   }
!   # also exclude unconfirmed users, otherwise they'll block the process
!   # will give quite different statistics than when used web based
! #  $req = Sql_Query("select id from {$tables["user"]} where !confirmed");
! #  while ($row = Sql_Fetch_Row($req)) {
! #    array_push($doneusers,$row[0]);
! #  }
!   if (sizeof($doneusers))
!     $exclusion = " and listuser.userid not in (".join(",",$doneusers).")";
!   if (USE_LIST_EXCLUDE) {
!     $excluded_lists = Sql_Fetch_Row_Query(sprintf('select data from %s where name = "excludelist" and id = %d',
!       $GLOBALS["tables"]["messagedata"],$messageid));
!     if (strlen($excluded_lists[0])) {
!       $req = Sql_Query(sprintf('select listuser.userid from %s as listuser where listid in (%s)',
!         $GLOBALS["tables"]["listuser"],$excluded_lists[0]));
!       while ($row = Sql_Fetch_Row($req)) {
!         array_push($skipusers,$row[0]);
        }
-       $query .= sprintf(' and listuser.listid not in (%s)',$excluded_lists[0]);
      }
!     if (sizeof($skipusers))
!       $exclusion .= " and listuser.userid not in (".join(",",$skipusers).")";
!   }
! 
!   $userconfirmed = ' and user.confirmed and !user.blacklisted ';
! 
!   $query = sprintf('select distinct user.id from
!     %s as listuser,
!     %s as user,
!     %s as listmessage
!     where
!     listmessage.messageid = %d and
!     listmessage.listid = listuser.listid and
!     user.id = listuser.userid %s %s %s',
!     $tables['listuser'],$tables["user"],$tables['listmessage'],
!     $messageid,
!     $userconfirmed,
!     $exclusion,
!     $user_attribute_query);
  
  
!   if (VERBOSE) {
!     output($query);
!   }
! 
!   $userids = Sql_query($query);
!   if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
  
!   # now we have all our users to send the message to
!   $num_users = Sql_affected_rows();
!   output($GLOBALS['I18N']->get('Found them').': '.$num_users.' '.$GLOBALS['I18N']->get('to process'));
!   setMessageData($messageid,'to process',$num_users);
! 
!   if ($num_per_batch) {
!     # send in batches of $num_per_batch users
!     $batch_total = $num_users;
!     if ($num_per_batch > 0) {
!       #$query .= sprintf(' limit 0,%d',$num_per_batch);
!       $userids = Sql_query("$query");
!       if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
!     } else {
!       output($GLOBALS['I18N']->get('No users to process for this batch'));
!       $userids = Sql_Query(sprintf('select * from %s where id = 0',$tables["user"]));
!     }
!   }
!   $affrows = Sql_Affected_Rows();
!   while ($userdata = Sql_fetch_row($userids)) {
!     if ($num_per_batch && $sent >= $num_per_batch) {
!       output($GLOBALS['I18N']->get('batch limit reached').": $sent ($num_per_batch)");
!       $GLOBALS["wait"] = $batch_period;
!       return;
!     }
!     $userid = $userdata[0];    # id of the user
!     $some = 1;
!     set_time_limit(120);
!     # check if we have been "killed"
!     $alive = checkLock($send_process_id);
!     if ($alive)
!       keepLock($send_process_id);
      else
!       ProcessError($GLOBALS['I18N']->get('Process Killed by other process'));
! 
!     # check if the message we are working on is still there and in process
!     $status = Sql_Fetch_Array_query("select id,status from {$tables['message']} where id = $messageid");
!     if (!$status['id']) {
!       ProcessError($GLOBALS['I18N']->get('Message I was working on has disappeared'));
!     } elseif ($status['status'] != 'inprocess') {
!       ProcessError($GLOBALS['I18N']->get('Sending of this message has been suspended'));
      }
  
!     flush();
  
!     # check whether the user has already received the message
!     $um = Sql_query("select entered from {$tables['usermessage']} where userid = $userdata[0] and messageid = $messageid");
!     if (!Sql_Affected_Rows()) {
!       if ($script_stage < 4)
!         $script_stage = 4; # we know a user
!       $someusers = 1;
!       $users = Sql_query("select id,email,uniqid,htmlemail,rssfrequency,confirmed,blacklisted from {$tables['user']} where id = $userid");
! 
!       # pick the first one (rather historical)
!       $user = Sql_fetch_row($users);
!       if ($user[5] && is_email($user[1])) {
!         $userid = $user[0];    # id of the user
!         $useremail = $user[1]; # email of the user
!         $userhash = $user[2];  # unique string of the user
!         $htmlpref = $user[3];  # preference for HTML emails
!         $rssfrequency = $user[4];
!         $confirmed = $user[5];
!         $blacklisted = $user[6];
! 
!         if (ENABLE_RSS && $processrss) {
!           if ($rssfrequency == $message["rsstemplate"]) {
!             # output("User matches message frequency");
!             $rssitems = rssUserHasContent($userid,$messageid,$rssfrequency);
!             $cansend = sizeof($rssitems) && (sizeof($rssitems) > $rss_content_treshold);
! #            if (!$cansend)
! #              output("No content to send for this user ".sizeof($rssitems));
            } else {
!             $cansend = 0;
            }
-         } else {
-           $cansend = !$blacklisted;
-         }
  
!         $throttled = 0;
!         if ($cansend && USE_DOMAIN_THROTTLE) {
!           list($mailbox,$domainname) = explode('@',$useremail);
!           $now = time();
!           $interval = $now - ($now % DOMAIN_BATCH_PERIOD);
!           if (!is_array($domainthrottle[$domainname])) {
!             $domainthrottle[$domainname] = array();
!           } elseif ($domainthrottle[$domainname]['interval'] == $interval) {
!             $throttled = $domainthrottle[$domainname]['sent'] >= DOMAIN_BATCH_SIZE;
!             if ($throttled) {
!               $domainthrottle[$domainname]['attempted']++;
!               if (DOMAIN_AUTO_THROTTLE
!                 && $domainthrottle[$domainname]['attempted'] > 25 # skip a few before auto throttling
!                 && $num_messages <= 1 # only do this when there's only one message to process otherwise the other ones don't get a change
!                 && $num_users < 1000 # and also when there's not too many left, because then it's likely they're all being throttled
!               ) {
!                 $domainthrottle[$domainname]['attempted'] = 0;
!                 logEvent(sprintf($GLOBALS['I18N']->get('There have been more than 10 attempts to send to %s that have been blocked for domain throttling.'),$domainname));
!                 logEvent($GLOBALS['I18N']->get('Introducing extra delay to decrease throttle failures'));
!                 if (VERBOSE) {
!                   output($GLOBALS['I18N']->get('Introducing extra delay to decrease throttle failures'));
!                 }
!                 if (!isset($running_throttle_delay)) {
!                   $running_throttle_delay = (int)(MAILQUEUE_THROTTLE + (DOMAIN_BATCH_PERIOD / (DOMAIN_BATCH_SIZE * 4)));
!                 } else {
!                   $running_throttle_delay += (int)(DOMAIN_BATCH_PERIOD / (DOMAIN_BATCH_SIZE * 4));
                  }
-                 #output("Running throttle delay: ".$running_throttle_delay);
-               } elseif (VERBOSE) {
-                 output(sprintf($GLOBALS['I18N']->get('%s is currently over throttle limit of %d per %d seconds').' ('.$domainthrottle[$domainname]['sent'].')',$domainname,DOMAIN_BATCH_SIZE,DOMAIN_BATCH_PERIOD));
                }
              }
            }
-         }
  
!         if ($cansend) {
!           $success = 0;
!           if (!TEST) {
!             if (!$throttled) {
!               if (VERBOSE)
!                 output($GLOBALS['I18N']->get('Sending').' '. $messageid.' '.$GLOBALS['I18N']->get('to').' '. $useremail);
!               $timer = new timer();
!               $success = sendEmail($messageid,$useremail,$userhash,$htmlpref,$rssitems);
!               if (VERBOSE) {
!                 output($GLOBALS['I18N']->get('It took').' '.$timer->elapsed(1).' '.$GLOBALS['I18N']->get('seconds to send'));
!               }
!             } else {
!               $throttlecount++;
!             }
!           } else {
!             $success = sendEmailTest($messageid,$useremail);
!           }
!           if ($success) {
!             if (USE_DOMAIN_THROTTLE) {
!               list($mailbox,$domainname) = explode('@',$useremail);
!               if ($domainthrottle[$domainname]['interval'] != $interval) {
!                 $domainthrottle[$domainname]['interval'] = $interval;
!                 $domainthrottle[$domainname]['sent']=0;
                } else {
!                 $domainthrottle[$domainname]['sent']++;
                }
              }
-             $sent++;
-             $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"sent\")");
-             if (ENABLE_RSS && $processrss) {
-               foreach ($rssitems as $rssitemid) {
-                 $status = Sql_query("update {$tables['rssitem']} set processed = processed +1 where id = $rssitemid");
-                 $um = Sql_query("replace into {$tables['rssitem_user']} (userid,itemid) values($userid,$rssitemid)");
-               }
-               Sql_Query("replace into {$tables["user_rss"]} (userid,last) values($userid,date_sub(now(),interval 15 minute))");
  
                }
!            } else {
!              $failed_sent++;
!              if (VERBOSE) {
!                output($GLOBALS['I18N']->get('Failed sending to').' '. $useremail);
!                logEvent("Failed sending message $messageid to $useremail");
!              }
!              # make sure it's not because it's an invalid email
!              # unconfirm this user, so they're not included next time
!              if (!validateEmail($useremail)) {
!                logEvent("invalid email $useremail user marked unconfirmed");
!                Sql_Query(sprintf('update %s set confirmed = 0 where email = "%s"',
!                  $GLOBALS['tables']['user'],$useremail));
               }
!            }
!            if ($script_stage < 5) {
!              $script_stage = 5; # we have actually sent one user
!            }
!            if (isset($running_throttle_delay)) {
!              sleep($running_throttle_delay);
!              if ($sent % 5 == 0) {
!                # retry running faster after some more messages, to see if that helps
!                unset($running_throttle_delay);
               }
!            } elseif (MAILQUEUE_THROTTLE) {
!              sleep(MAILQUEUE_THROTTLE);
!            } elseif (MAILQUEUE_BATCH_SIZE && MAILQUEUE_AUTOTHROTTLE && $sent > 10) {
!              $totaltime = $GLOBALS['processqueue_timer']->elapsed(1);
!              $msgperhour = (3600/$totaltime) * $sent;
!              $msgpersec = $msgperhour / 3600;
!              $secpermsg = $totaltime / $sent;
!              $target = MAILQUEUE_BATCH_SIZE / MAILQUEUE_BATCH_PERIOD;
!              $actual = $sent / $totaltime;
!              $delay = $actual - $target;
!   #           output("Sent: $sent mph $msgperhour mps $msgpersec secpm $secpermsg target $target actual $actual d $delay");
!              if ($delay > 0) {
!   #             $expected = MAILQUEUE_BATCH_PERIOD / $secpermsg;
!   #             $delay = MAILQUEUE_BATCH_SIZE / $expected;
!                if (VERBOSE) {
!                  output($GLOBALS['I18N']->get('waiting for').' '.$delay.' '.$GLOBALS['I18N']->get('seconds').' '.
!                    $GLOBALS['I18N']->get('to make sure we don\'t exceed our limit of').MAILQUEUE_BATCH_SIZE.' '.
!                    $GLOBALS['I18N']->get('messages in').' '.MAILQUEUE_BATCH_PERIOD.$GLOBALS['I18N']->get('seconds'));
                 }
-                $delay = $delay * 1000000;
-                usleep($delay);
               }
!            }
!         } else {
!           $cannotsend++;
!           # mark it as sent anyway, because otherwise the process will never finish
!           if (VERBOSE) {
!             output($GLOBALS['I18N']->get('not sending to ').$useremail);
            }
-           $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"not sent\")");
-         }
  
!         # update possible other users matching this email as well,
!         # to avoid duplicate sending when people have subscribed multiple times
!         # bit of legacy code after making email unique in the database
! #        $emails = Sql_query("select * from {$tables['user']} where email =\"$useremail\"");
! #        while ($email = Sql_fetch_row($emails))
! #          Sql_query("replace into {$tables['usermessage']} (userid,messageid) values($email[0],$messageid)");
!       }  else {
!         # some "invalid emails" are entirely empty, ah, that is because they are unconfirmed
! 
!         ## this is quite old as well, with the preselection that avoids unconfirmed users
!         # it is unlikely this is every processed.
! 
!         if (!$user[5]) {
!           if (VERBOSE)
!             output($GLOBALS['I18N']->get('Unconfirmed user').': '."$userid $user[1], $user[0]");
!           $unconfirmed++;
!           # when running from commandline we mark it as sent, otherwise we might get
!           # stuck when using batch processing
!          # if ($GLOBALS["commandline"]) {
!             $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"unconfirmed user\")");
!          # }
!         } elseif ($user[1] || $user[0]) {
!           if (VERBOSE)
!             output("Invalid email: $user[1], $user[0]");
!           logEvent("Invalid email, userid $user[0], email $user[1]");
!           # mark it as sent anyway
!           if ($userid)
!             $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"invalid email\")");
!           $invalid++;
          }
!       }
!     } else {
  
!       ## and this is quite historical, and also unlikely to be every called
!       # because we now exclude users who have received the message from the
!       # query to find users to send to
! 
!       $um = Sql_Fetch_Row($um);
!       $notsent++;
!       if (VERBOSE)
!         output($GLOBALS['I18N']->get('Not sending to').' '. $userdata[0].', '.$GLOBALS['I18N']->get('already sent').' '.$um[0]);
      }
-     $status = Sql_query("update {$tables['message']} set processed = processed +1 where id = $messageid");
      $processed = $notsent + $sent + $invalid + $unconfirmed + $cannotsend + $failed_sent;
!     #if ($processed % 10 == 0) {
!     if (0) {
!       output('AR'.$affrows.' N '.$num_users.' P'.$processed.' S'.$sent.' N'.$notsent.' I'.$invalid.' U'.$unconfirmed.' C'.$cannotsend.' F'.$failed_sent);
!       $rn = $reload * $num_per_batch;
!       output('P '.$processed .' N'. $num_users .' NB'.$num_per_batch .' BT'.$batch_total .' R'.$reload.' RN'.$rn);
!     }
!     $totaltime = $GLOBALS['processqueue_timer']->elapsed(1);
!     $msgperhour = (3600/$totaltime) * $sent;
!     $secpermsg = $totaltime / $sent;
!     $timeleft = ($num_users - $sent) * $secpermsg;
!     $eta = date('D j M H:i',time()+$timeleft);
!     setMessageData($messageid,'ETA',$eta);
!     setMessageData($messageid,'msg/hr',$msgperhour);
!     setMessageData($messageid,'to process',$num_users - $sent);
!   }
!   $processed = $notsent + $sent + $invalid + $unconfirmed + $cannotsend + $failed_sent;
!   output($GLOBALS['I18N']->get('Processed').' '. $processed.' '.$GLOBALS['I18N']->get('out of').' '. $num_users .' '.$GLOBALS['I18N']->get('users'));
!   if ($num_users - $sent <= 0) {
!     # this message is done
!     if (!$someusers)
!       output($GLOBALS['I18N']->get('Hmmm, No users found to send to'));
!     if (!$failed_sent) {
!       repeatMessage($messageid);
!       $status = Sql_query(sprintf('update %s set status = "sent",sent = now() where id = %d',$GLOBALS['tables']['message'],$messageid));
!       if ($msgdata['notify_end'] && !isset($msgdata['end_notified'])) {
!         $notifications = explode(',',$msgdata['notify_end']);
!         foreach ($notifications as $notification) {
!           sendMail($notification,$GLOBALS['I18N']->get('Message Sending has finished'),
!             sprintf($GLOBALS['I18N']->get('phplist has finished sending the message with subject %s'),$message['subject']));
          }
!         Sql_Query(sprintf('insert ignore into %s (name,id,data) values("end_notified",%d,now())',
!           $GLOBALS['tables']['messagedata'],$messageid));
        }
!       $timetaken = Sql_Fetch_Row_query("select sent,sendstart from {$tables['message']} where id = \"$messageid\"");
!       output($GLOBALS['I18N']->get('It took').' '.timeDiff($timetaken[0],$timetaken[1]).' '.$GLOBALS['I18N']->get('to send this message'));
!       sendMessageStats($messageid);
!     }
!   } else {
!     if ($script_stage < 5)
!       $script_stage = 5;
    }
- }
  
! if (!$num_messages)
!   $script_stage = 6; # we are done
! # shutdown will take care of reporting
  
  ?>
--- 306,796 ----
    output($GLOBALS['I18N']->get('Skipped in last run').": $lastskipped");
  }
  
! /* Process messages in two passes
! ** First pass - all sequenced (timed) messages
! ** Second pass - standard phpList messages
! **/
! 
! for ($pass=1; $pass <=2; $pass++)
! {
! 
!   output("Processing pass #$pass");
!   $script_stage = 1; # we are active
!   $notsent = $sent = $invalid = $unconfirmed = $cannotsend = 0;
! 
!   if ($pass == 1)
!     $messages = Sql_query("select distinct id,userselection,rsstemplate,subject from {$tables['message']} M, {$tables['usermessage_seq']} usermessage_seq where M.id = usermessage_seq.messageid and usermessage_seq.embargo < now()");
!   else  
!     $messages = Sql_query("select id,userselection,rsstemplate,subject from ".$tables["message"]." where status != \"draft\" and status != \"sent\" and status != \"prepared\" and status != \"suspended\" and embargo < now() order by entered");
  
!   $num_messages = Sql_affected_rows();
!   if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
! 
!   if ($num_messages) {
!     output($GLOBALS['I18N']->get('Processing has started,').' '.$num_messages.' '.$GLOBALS['I18N']->get('message(s) to process.'));
!     if (!$GLOBALS["commandline"]) {
!       if (!$safemode) {
!         if (!$num_per_batch) {
!           output($GLOBALS['I18N']->get('It is safe to click your stop button now, report will be sent by email to').' '.getConfig("report_address"));
!         } else {
!           output($GLOBALS['I18N']->get('Please leave this window open. You have batch processing enabled, so it will reload several times to send the messages. Reports will be sent by email to').' '.getConfig("report_address"));
!         }
        } else {
!         output($GLOBALS['I18N']->get('Your webserver is running in safe_mode. Please keep this window open. It may reload several times to make sure all messages are sent.').' '.$GLOBALS['I18N']->get('Reports will be sent by email to').' '.getConfig("report_address"));
        }
      }
    }
  
!   Sql_query("SET SQL_BIG_TABLES=1");
!   $script_stage = 2; # we know the messages to process
!   include_once "footer.inc";
!   if (!$num_per_batch) {
!     $num_per_batch = 1000000;
!   }
! 
!   while ($message = Sql_fetch_array($messages)) {
!     $failed_sent = 0;
!     $throttlecount = 0;
! 
!     $messageid = $message["id"];
!     $userselection = $message["userselection"];
!     $rssmessage = $message["rsstemplate"];
! 
!     $msgdata = loadMessageData($messageid);
!     if (!empty($msgdata['notify_start']) && !isset($msgdata['start_notified'])) {
!       $notifications = explode(',',$msgdata['notify_start']);
!       foreach ($notifications as $notification) {
!         sendMail($notification,$GLOBALS['I18N']->get('Message Sending has started'),
!           sprintf($GLOBALS['I18N']->get('phplist has started sending the message with subject %s'),$message['subject']."\n".
!           sprintf($GLOBALS['I18N']->get('to view the progress of this message, go to %s'),getConfig('website').$GLOBALS['adminpages'].'/?page=messages&type=sent')));
!       }
!       Sql_Query(sprintf('insert ignore into %s (name,id,data) values("start_notified",%d,now())',
!         $GLOBALS['tables']['messagedata'],$messageid));
      }
  
!     output($GLOBALS['I18N']->get('Processing message').' '. $messageid);
!     if (ENABLE_RSS && $message["rsstemplate"]) {
!       $processrss = 1;
!       output($GLOBALS['I18N']->get('Message').' '. $messageid.' '.$GLOBALS['I18N']->get('is an RSS feed for').' '. $GLOBALS['I18N']->get($rssmessage));
!     } else {
!       $processrss = 0;
      }
  
!     flush();
!     keepLock($send_process_id);
!     $status = Sql_query('update '.$tables["message"].' set status = "inprocess" where id = '.$messageid);
!     $sendstart = Sql_query('update '.$tables["message"].' set sendstart = now() where sendstart is NULL and id = '.$messageid);
!     output($GLOBALS['I18N']->get('Looking for users'));
!     if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
! 
!     # make selection on attribute, users who at least apply to the attributes
!     # lots of ppl seem to use it as a normal mailinglist system, and do not use attributes.
!     # Check this and take anyone in that case.
!     $numattr = Sql_Fetch_Row_Query("select count(*) from ".$tables["attribute"]);
! 
!     if ($userselection && $numattr[0]) {
!       $res = Sql_query($userselection);
!       $num_users = Sql_Affected_rows($res);
!       output($num_users.' '.$GLOBALS['I18N']->get('users apply for attributes, now checking lists'));
!       $user_list = "";
!       while ($row = Sql_Fetch_row($res)) {
!         $user_list .= $row[0] . ",";
!       }
!       $user_list = substr($user_list,0,-1);
!       if ($user_list)
!         $user_attribute_query = " and listuser.userid in ($user_list)";
!       else {
!         output($GLOBALS['I18N']->get('No users apply for attributes'));
!         $status = Sql_query("update {$tables["message"]} set status = \"sent\",sent = now() where id = \"$messageid\"");
!         finish("info","Message $messageid: \nNo users apply for attributes, ie nothing to do");
!         $script_stage = 6;
!         # we should actually continue with the next message
!         continue;
        }
      }
!     if ($script_stage < 3)
!       $script_stage = 3; # we know the users by attribute
  
+     # when using commandline we need to exclude users who have already received
+     # the email
+     # we don't do this otherwise because it slows down the process, possibly
+     # causing us to not find anything at all
+     $exclusion = "";
+     $doneusers = array();
+     $skipusers = array();
+     if (VERBOSE) {
+       output($GLOBALS['I18N']->get('looking for users who can be excluded from this mailing'));
+     }
+ 
+     /* Find out which users have ALREADY received message, so they can be excluded */
+     $req = Sql_Query("select userid from {$tables["usermessage"]} where messageid = $messageid");
+     $skipped = Sql_Affected_Rows();
+     while ($row = Sql_Fetch_Row($req)) {
+       $alive = checkLock($send_process_id);
+       if ($alive)
+         keepLock($send_process_id);
+       else
+         ProcessError($GLOBALS['I18N']->get('Process Killed by other process'));
+       array_push($doneusers,$row[0]);
+     }
+     # also exclude unconfirmed users, otherwise they'll block the process
+     # will give quite different statistics than when used web based
+   #  $req = Sql_Query("select id from {$tables["user"]} where !confirmed");
+   #  while ($row = Sql_Fetch_Row($req)) {
+   #    array_push($doneusers,$row[0]);
+   #  }
+     if (sizeof($doneusers))
+       $exclusion = " and user.id not in (".join(",",$doneusers).")";
+     if (USE_LIST_EXCLUDE) {
+       $excluded_lists = Sql_Fetch_Row_Query(sprintf('select data from %s where name = "excludelist" and id = %d',
+         $GLOBALS["tables"]["messagedata"],$messageid));
+       if (strlen($excluded_lists[0])) {
+         $req = Sql_Query(sprintf('select listuser.userid from %s as listuser where listid in (%s)',
+           $GLOBALS["tables"]["listuser"],$excluded_lists[0]));
+         while ($row = Sql_Fetch_Row($req)) {
+           array_push($skipusers,$row[0]);
+         }
+         $query .= sprintf(' and listuser.listid not in (%s)',$excluded_lists[0]);
+       }
+       if (sizeof($skipusers))
+         $exclusion .= " and user.id not in (".join(",",$skipusers).")";
+     }
  
!     $userconfirmed = ' and user.confirmed and !user.blacklisted ';
  
!     if ($pass==1)
!       $query = sprintf('select distinct user.id from
!         %s as user,
!         %s as usermessage_seq
!         where
!         usermessage_seq.messageid = %d and
!         user.id = usermessage_seq.userid %s %s',
!         $tables["user"],$tables['usermessage_seq'],
!         $messageid,
!         $userconfirmed,
!         $exclusion,
!         $user_attribute_query); 
      else
!       $query = sprintf('select distinct user.id from
!         %s as listuser,
!         %s as user,
!         %s as listmessage
!         where
!         listmessage.messageid = %d and
!         listmessage.listid = listuser.listid and
!         user.id = listuser.userid %s %s %s',
!         $tables['listuser'],$tables["user"],$tables['listmessage'],
!         $messageid,
!         $userconfirmed,
!         $exclusion,
!         $user_attribute_query);
! 
! 
!     if (VERBOSE) {
!       output($query);
!     }
! 
!     $userids = Sql_query($query);
!     if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
! 
!     # now we have all our users to send the message to
!     $num_users = Sql_affected_rows();
!     output($GLOBALS['I18N']->get('Found them').': '.$num_users.' '.$GLOBALS['I18N']->get('to process'));
!     setMessageData($messageid,'to process',$num_users);
! 
!     if ($num_per_batch) {
!       # send in batches of $num_per_batch users
!       $batch_total = $num_users;
!       if ($num_per_batch > 0) {
!         #$query .= sprintf(' limit 0,%d',$num_per_batch);
!         $userids = Sql_query("$query");
!         if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
!       } else {
!         output($GLOBALS['I18N']->get('No users to process for this batch'));
!         $userids = Sql_Query(sprintf('select * from %s where id = 0',$tables["user"]));
!       }
      }
+     $affrows = Sql_Affected_Rows();
+     while ($userdata = Sql_fetch_row($userids)) {
+       if ($num_per_batch && $sent >= $num_per_batch) {
+         output($GLOBALS['I18N']->get('batch limit reached').": $sent ($num_per_batch)");
+         $GLOBALS["wait"] = $batch_period;
+         return;
+       }
+       $userid = $userdata[0];    # id of the user
+       $some = 1;
+       set_time_limit(120);
+       # check if we have been "killed"
+       $alive = checkLock($send_process_id);
+       if ($alive)
+         keepLock($send_process_id);
+       else
+         ProcessError($GLOBALS['I18N']->get('Process Killed by other process'));
+ 
+       # check if the message we are working on is still there and in process
+       $status = Sql_Fetch_Array_query("select id,status from {$tables['message']} where id = $messageid");
+       if (!$status['id']) {
+         ProcessError($GLOBALS['I18N']->get('Message I was working on has disappeared'));
+       } elseif ($status['status'] != 'inprocess') {
+         ProcessError($GLOBALS['I18N']->get('Sending of this message has been suspended'));
+       }
  
!       flush();
  
!       # check whether the user has already received the message
!       $um = Sql_query("select entered from {$tables['usermessage']} where userid = $userdata[0] and messageid = $messageid");
!       if (!Sql_Affected_Rows()) {
!         if ($script_stage < 4)
!           $script_stage = 4; # we know a user
!         $someusers = 1;
!         $users = Sql_query("select id,email,uniqid,htmlemail,rssfrequency,confirmed,blacklisted from {$tables['user']} where id = $userid");
! 
!         # pick the first one (rather historical)
!         $user = Sql_fetch_row($users);
!         if ($user[5] && is_email($user[1])) {
!           $userid = $user[0];    # id of the user
!           $useremail = $user[1]; # email of the user
!           $userhash = $user[2];  # unique string of the user
!           $htmlpref = $user[3];  # preference for HTML emails
!           $rssfrequency = $user[4];
!           $confirmed = $user[5];
!           $blacklisted = $user[6];
! 
!           if (ENABLE_RSS && $processrss) {
!             if ($rssfrequency == $message["rsstemplate"]) {
!               # output("User matches message frequency");
!               $rssitems = rssUserHasContent($userid,$messageid,$rssfrequency);
!               $cansend = sizeof($rssitems) && (sizeof($rssitems) > $rss_content_treshold);
!   #            if (!$cansend)
!   #              output("No content to send for this user ".sizeof($rssitems));
!             } else {
!               $cansend = 0;
!             }
            } else {
!             $cansend = !$blacklisted;
            }
  
!           $throttled = 0;
!           if ($cansend && USE_DOMAIN_THROTTLE) {
!             list($mailbox,$domainname) = explode('@',$useremail);
!             $now = time();
!             $interval = $now - ($now % DOMAIN_BATCH_PERIOD);
!             if (!is_array($domainthrottle[$domainname])) {
!               $domainthrottle[$domainname] = array();
!             } elseif ($domainthrottle[$domainname]['interval'] == $interval) {
!               $throttled = $domainthrottle[$domainname]['sent'] >= DOMAIN_BATCH_SIZE;
!               if ($throttled) {
!                 $domainthrottle[$domainname]['attempted']++;
!                 if (DOMAIN_AUTO_THROTTLE
!                   && $domainthrottle[$domainname]['attempted'] > 25 # skip a few before auto throttling
!                   && $num_messages <= 1 # only do this when there's only one message to process otherwise the other ones don't get a change
!                   && $num_users < 1000 # and also when there's not too many left, because then it's likely they're all being throttled
!                 ) {
!                   $domainthrottle[$domainname]['attempted'] = 0;
!                   logEvent(sprintf($GLOBALS['I18N']->get('There have been more than 10 attempts to send to %s that have been blocked for domain throttling.'),$domainname));
!                   logEvent($GLOBALS['I18N']->get('Introducing extra delay to decrease throttle failures'));
!                   if (VERBOSE) {
!                     output($GLOBALS['I18N']->get('Introducing extra delay to decrease throttle failures'));
!                   }
!                   if (!isset($running_throttle_delay)) {
!                     $running_throttle_delay = (int)(MAILQUEUE_THROTTLE + (DOMAIN_BATCH_PERIOD / (DOMAIN_BATCH_SIZE * 4)));
!                   } else {
!                     $running_throttle_delay += (int)(DOMAIN_BATCH_PERIOD / (DOMAIN_BATCH_SIZE * 4));
!                   }
!                   #output("Running throttle delay: ".$running_throttle_delay);
!                 } elseif (VERBOSE) {
!                   output(sprintf($GLOBALS['I18N']->get('%s is currently over throttle limit of %d per %d seconds').' ('.$domainthrottle[$domainname]['sent'].')',$domainname,DOMAIN_BATCH_SIZE,DOMAIN_BATCH_PERIOD));
                  }
                }
              }
            }
  
!           if ($cansend) {
!             $success = 0;
!             if (!TEST) {
!               if (!$throttled) {
!                 if (VERBOSE)
!                   output($GLOBALS['I18N']->get('Sending').' '. $messageid.' '.$GLOBALS['I18N']->get('to').' '. $useremail);
!                 $timer = new timer();
!                 $success = sendEmail($messageid,$useremail,$userhash,$htmlpref,$rssitems);
!                 if (VERBOSE) {
!                   output($GLOBALS['I18N']->get('It took').' '.$timer->elapsed(1).' '.$GLOBALS['I18N']->get('seconds to send'));
!                 }
                } else {
!                 $throttlecount++;
                }
+             } else {
+               $success = sendEmailTest($messageid,$useremail);
              }
  
+             /* Sent queued sequential e-mail, remove from queue */
+             if ($pass == 1)
+               Sql_query("delete * from {$tables['usermessage_seq']} where userid=$userid and messageid=$messageid");
+             if ($success) {
+               if (USE_DOMAIN_THROTTLE) {
+                 list($mailbox,$domainname) = explode('@',$useremail);
+                 if ($domainthrottle[$domainname]['interval'] != $interval) {
+                   $domainthrottle[$domainname]['interval'] = $interval;
+                   $domainthrottle[$domainname]['sent']=0;
+                 } else {
+                   $domainthrottle[$domainname]['sent']++;
+                 }
                }
!               $sent++;
!               $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"sent\")");
!               if (ENABLE_RSS && $processrss) {
!                 foreach ($rssitems as $rssitemid) {
!                   $status = Sql_query("update {$tables['rssitem']} set processed = processed +1 where id = $rssitemid");
!                   $um = Sql_query("replace into {$tables['rssitem_user']} (userid,itemid) values($userid,$rssitemid)");
!                 }
!                 Sql_Query("replace into {$tables["user_rss"]} (userid,last) values($userid,date_sub(now(),interval 15 minute))");
! 
!                 }
!              } else {
!                $failed_sent++;
!                if (VERBOSE) {
!                  output($GLOBALS['I18N']->get('Failed sending to').' '. $useremail);
!                  logEvent("Failed sending message $messageid to $useremail");
!                }
!                # make sure it's not because it's an invalid email
!                # unconfirm this user, so they're not included next time
!                if (!validateEmail($useremail)) {
!                  logEvent("invalid email $useremail user marked unconfirmed");
!                  Sql_Query(sprintf('update %s set confirmed = 0 where email = "%s"',
!                    $GLOBALS['tables']['user'],$useremail));
!                }
               }
!              if ($script_stage < 5) {
!                $script_stage = 5; # we have actually sent one user
               }
!              if (isset($running_throttle_delay)) {
!                sleep($running_throttle_delay);
!                if ($sent % 5 == 0) {
!                  # retry running faster after some more messages, to see if that helps
!                  unset($running_throttle_delay);
!                }
!              } elseif (MAILQUEUE_THROTTLE) {
!                sleep(MAILQUEUE_THROTTLE);
!              } elseif (MAILQUEUE_BATCH_SIZE && MAILQUEUE_AUTOTHROTTLE && $sent > 10) {
!                $totaltime = $GLOBALS['processqueue_timer']->elapsed(1);
!                $msgperhour = (3600/$totaltime) * $sent;
!                $msgpersec = $msgperhour / 3600;
!                $secpermsg = $totaltime / $sent;
!                $target = MAILQUEUE_BATCH_SIZE / MAILQUEUE_BATCH_PERIOD;
!                $actual = $sent / $totaltime;
!                $delay = $actual - $target;
!     #           output("Sent: $sent mph $msgperhour mps $msgpersec secpm $secpermsg target $target actual $actual d $delay");
!                if ($delay > 0) {
!     #             $expected = MAILQUEUE_BATCH_PERIOD / $secpermsg;
!     #             $delay = MAILQUEUE_BATCH_SIZE / $expected;
!                  if (VERBOSE) {
!                    output($GLOBALS['I18N']->get('waiting for').' '.$delay.' '.$GLOBALS['I18N']->get('seconds').' '.
!                      $GLOBALS['I18N']->get('to make sure we don\'t exceed our limit of').MAILQUEUE_BATCH_SIZE.' '.
!                      $GLOBALS['I18N']->get('messages in').' '.MAILQUEUE_BATCH_PERIOD.$GLOBALS['I18N']->get('seconds'));
!                  }
!                  $delay = $delay * 1000000;
!                  usleep($delay);
                 }
               }
!           } else {
!             $cannotsend++;
!             # mark it as sent anyway, because otherwise the process will never finish
!             if (VERBOSE) {
!               output($GLOBALS['I18N']->get('not sending to ').$useremail);
!             }
!             $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"not sent\")");
            }
  
!           # update possible other users matching this email as well,
!           # to avoid duplicate sending when people have subscribed multiple times
!           # bit of legacy code after making email unique in the database
!   #        $emails = Sql_query("select * from {$tables['user']} where email =\"$useremail\"");
!   #        while ($email = Sql_fetch_row($emails))
!   #          Sql_query("replace into {$tables['usermessage']} (userid,messageid) values($email[0],$messageid)");
!         }  else {
!           # some "invalid emails" are entirely empty, ah, that is because they are unconfirmed
! 
!           ## this is quite old as well, with the preselection that avoids unconfirmed users
!           # it is unlikely this is every processed.
! 
!           if (!$user[5]) {
!             if (VERBOSE)
!               output($GLOBALS['I18N']->get('Unconfirmed user').': '."$userid $user[1], $user[0]");
!             $unconfirmed++;
!             # when running from commandline we mark it as sent, otherwise we might get
!             # stuck when using batch processing
!            # if ($GLOBALS["commandline"]) {
!               $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"unconfirmed user\")");
!            # }
!           } elseif ($user[1] || $user[0]) {
!             if (VERBOSE)
!               output("Invalid email: $user[1], $user[0]");
!             logEvent("Invalid email, userid $user[0], email $user[1]");
!             # mark it as sent anyway
!             if ($userid)
!               $um = Sql_query("replace into {$tables['usermessage']} (entered,userid,messageid,status) values(now(),$userid,$messageid,\"invalid email\")");
!             $invalid++;
!           }
          }
!       } else {
  
!         ## and this is quite historical, and also unlikely to be every called
!         # because we now exclude users who have received the message from the
!         # query to find users to send to
! 
!         $um = Sql_Fetch_Row($um);
!         $notsent++;
!         if (VERBOSE)
!           output($GLOBALS['I18N']->get('Not sending to').' '. $userdata[0].', '.$GLOBALS['I18N']->get('already sent').' '.$um[0]);
!       }
!       $status = Sql_query("update {$tables['message']} set processed = processed +1 where id = $messageid");
!       $processed = $notsent + $sent + $invalid + $unconfirmed + $cannotsend + $failed_sent;
!       #if ($processed % 10 == 0) {
!       if (0) {
!         output('AR'.$affrows.' N '.$num_users.' P'.$processed.' S'.$sent.' N'.$notsent.' I'.$invalid.' U'.$unconfirmed.' C'.$cannotsend.' F'.$failed_sent);
!         $rn = $reload * $num_per_batch;
!         output('P '.$processed .' N'. $num_users .' NB'.$num_per_batch .' BT'.$batch_total .' R'.$reload.' RN'.$rn);
!       }
!       $totaltime = $GLOBALS['processqueue_timer']->elapsed(1);
!       $msgperhour = (3600/$totaltime) * $sent;
!       $secpermsg = $totaltime / $sent;
!       $timeleft = ($num_users - $sent) * $secpermsg;
!       $eta = date('D j M H:i',time()+$timeleft);
!       setMessageData($messageid,'ETA',$eta);
!       setMessageData($messageid,'msg/hr',$msgperhour);
!       setMessageData($messageid,'to process',$num_users - $sent);
      }
      $processed = $notsent + $sent + $invalid + $unconfirmed + $cannotsend + $failed_sent;
!     output($GLOBALS['I18N']->get('Processed').' '. $processed.' '.$GLOBALS['I18N']->get('out of').' '. $num_users .' '.$GLOBALS['I18N']->get('users'));
!     if ($num_users - $sent <= 0) {
!       # this message is done
!       if (!$someusers)
!         output($GLOBALS['I18N']->get('Hmmm, No users found to send to'));
!       if (!$failed_sent) {
!         repeatMessage($messageid);
!         $status = Sql_query(sprintf('update %s set status = "sent",sent = now() where id = %d',$GLOBALS['tables']['message'],$messageid));
!         if ($msgdata['notify_end'] && !isset($msgdata['end_notified'])) {
!           $notifications = explode(',',$msgdata['notify_end']);
!           foreach ($notifications as $notification) {
!             sendMail($notification,$GLOBALS['I18N']->get('Message Sending has finished'),
!               sprintf($GLOBALS['I18N']->get('phplist has finished sending the message with subject %s'),$message['subject']));
!           }
!           Sql_Query(sprintf('insert ignore into %s (name,id,data) values("end_notified",%d,now())',
!             $GLOBALS['tables']['messagedata'],$messageid));
          }
!         $timetaken = Sql_Fetch_Row_query("select sent,sendstart from {$tables['message']} where id = \"$messageid\"");
!         output($GLOBALS['I18N']->get('It took').' '.timeDiff($timetaken[0],$timetaken[1]).' '.$GLOBALS['I18N']->get('to send this message'));
!         sendMessageStats($messageid);
        }
!     } else {
!       if ($script_stage < 5)
!         $script_stage = 5;
!     }
    }
  
!   if (!$num_messages)
!     $script_stage = 6; # we are done
!   # shutdown will take care of reporting
! }
  
  ?>
Index: admin/structure.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/structure.php,v
retrieving revision 1.9
diff -c -r1.9 structure.php
*** admin/structure.php	3 Aug 2005 02:36:59 -0000	1.9
--- admin/structure.php	1 Aug 2006 12:02:33 -0000
***************
*** 132,138 ****
          "listid" => array("integer not null","List ID"),
          "entered" => array("datetime", "Entered"),
          "modified" => array("timestamp","Modified"),
!         "unique" => array("(messageid,listid)","")
      ),
      "rssitem" => array(
          "id" => array("integer not null primary key auto_increment","ID"),
--- 132,139 ----
          "listid" => array("integer not null","List ID"),
          "entered" => array("datetime", "Entered"),
          "modified" => array("timestamp","Modified"),
!         "unique" => array("(messageid,listid)",""),
!         "sequence_hour" => array("float default null", "Automatically deliver messages n hours after user has signed up for a list")
      ),
      "rssitem" => array(
          "id" => array("integer not null primary key auto_increment","ID"),
***************
*** 187,192 ****
--- 188,200 ----
          "index" => array("useridindex (userid)",""),
          "index" => array("enteredindex (entered)",""),
      ),
+     "usermessage_seq" => array ( # linking messages to a user
+         #"id" => array("integer not null primary key auto_increment","ID"),
+         "messageid" => array("integer not null","Message ID"),
+         "userid" => array("integer not null","User ID"),
+         "embargo" => array("datetime", "Time to send message"),
+         "primary key" => array("(userid,messageid)", "Pkey")
+     ),
      "sendprocess" => array( # keep track of running send processes to avoid to many running concurrently
          "id" => array("integer not null primary key auto_increment","ID"),
          "started" => array("datetime", "Start Time"),
Index: admin/subscribelib2.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/subscribelib2.php,v
retrieving revision 1.21
diff -c -r1.21 subscribelib2.php
*** admin/subscribelib2.php	2 Dec 2005 00:57:15 -0000	1.21
--- admin/subscribelib2.php	1 Aug 2006 12:02:35 -0000
***************
*** 98,104 ****
    $validhost = 1;
  }
  
! $listsok = ((!ALLOW_NON_LIST_SUBSCRIBE && is_array($_POST["list"])) || ALLOW_NON_LIST_SUBSCRIBE);
  
  if (isset($_POST["subscribe"]) && is_email($_POST["email"]) && $listsok
     && $allthere && $validhost) {
--- 98,104 ----
    $validhost = 1;
  }
  
! $listsok = ((!ALLOW_NON_LIST_SUBSCRIBE && isset($_POST["list"]) &&is_array($_POST["list"])) || ALLOW_NON_LIST_SUBSCRIBE);
  
  if (isset($_POST["subscribe"]) && is_email($_POST["email"]) && $listsok
     && $allthere && $validhost) {
***************
*** 116,122 ****
      $email = $regs[1];
    }
    $result = Sql_query("select * from {$GLOBALS["tables"]["user"]} where email = \"$email\"");#"
!   $rssfrequency = validateRssFrequency($_POST['rssfrequency']);
  
    if (!Sql_affected_rows()) {
      # they do not exist, so add them
--- 116,122 ----
      $email = $regs[1];
    }
    $result = Sql_query("select * from {$GLOBALS["tables"]["user"]} where email = \"$email\"");#"
!   $rssfrequency = isset($_POST['rssfrequency']) ? validateRssFrequency($_POST['rssfrequency']) : '';
  
    if (!Sql_affected_rows()) {
      # they do not exist, so add them
Index: admin/upgrade.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/upgrade.php,v
retrieving revision 1.16
diff -c -r1.16 upgrade.php
*** admin/upgrade.php	2 Dec 2005 00:57:15 -0000	1.16
--- admin/upgrade.php	1 Aug 2006 12:02:37 -0000
***************
*** 303,308 ****
--- 303,313 ----
          Sql_Create_Table($tables["bounceregex"],$DBstruct["bounceregex"]);
          Sql_Create_Table($tables["bounceregex_bounce"],$DBstruct["bounceregex_bounce"]);
        }
+ 
+       if ($minor <9 || ($minor == 9 && $sub < 6)) {
+         Sql_Create_Table($tables["usermessage_seq"], $DBStruct["usermessage_seq"]);
+         Sql_Query("alter table {$tables['listmessage']} add sequence_hour float default null");
+       }
        break;
    }
  

Index: index.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/index.php,v
retrieving revision 1.12
diff -r1.12 index.php
31a32,33
> include_once dirname(__FILE__)."/admin/sequence.php";
> 
460a463
>     trigger_user_confirmed($userdata["id"]);

Index: admin/processqueue.php
===================================================================
RCS file: /cvsroot/phplist/phplist/public_html/lists/admin/processqueue.php,v
retrieving revision 1.19
diff -b -B -c -r1.19 processqueue.php
*** admin/processqueue.php	2 Dec 2005 00:57:15 -0000	1.19
--- admin/processqueue.php	1 Aug 2006 12:33:03 -0000
***************
*** 306,319 ****
    output($GLOBALS['I18N']->get('Skipped in last run').": $lastskipped");
  }
  
! $script_stage = 1; # we are active
! $notsent = $sent = $invalid = $unconfirmed = $cannotsend = 0;
  
! $messages = Sql_query("select id,userselection,rsstemplate,subject from ".$tables["message"]." where status != \"draft\" and status != \"sent\" and status != \"prepared\" and status != \"suspended\" and embargo < now() order by entered");
! $num_messages = Sql_affected_rows();
! if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
  
! if ($num_messages) {
    output($GLOBALS['I18N']->get('Processing has started,').' '.$num_messages.' '.$GLOBALS['I18N']->get('message(s) to process.'));
    if (!$GLOBALS["commandline"]) {
      if (!$safemode) {
--- 306,332 ----
    output($GLOBALS['I18N']->get('Skipped in last run').": $lastskipped");
  }
  
! /* Process messages in two passes
! ** First pass - all sequenced (timed) messages
! ** Second pass - standard phpList messages
! **/
! 
! for ($pass=1; $pass <=2; $pass++)
! {
! 
!   output("Processing pass #$pass");
!   $script_stage = 1; # we are active
!   $notsent = $sent = $invalid = $unconfirmed = $cannotsend = 0;
  
!   if ($pass == 1)
!     $messages = Sql_query("select distinct id,userselection,rsstemplate,subject from {$tables['message']} M, {$tables['usermessage_seq']} usermessage_seq where M.id = usermessage_seq.messageid and usermessage_seq.embargo < now()");
!   else  
!     $messages = Sql_query("select id,userselection,rsstemplate,subject from ".$tables["message"]." where status != \"draft\" and status != \"sent\" and status != \"prepared\" and status != \"suspended\" and embargo < now() order by entered");
! 
!   $num_messages = Sql_affected_rows();
!   if (Sql_Has_Error($database_connection)) {  ProcessError(Sql_Error($database_connection)); }
  
!   if ($num_messages) {
      output($GLOBALS['I18N']->get('Processing has started,').' '.$num_messages.' '.$GLOBALS['I18N']->get('message(s) to process.'));
      if (!$GLOBALS["commandline"]) {
        if (!$safemode) {
***************
*** 326,341 ****
        output($GLOBALS['I18N']->get('Your webserver is running in safe_mode. Please keep this window open. It may reload several times to make sure all messages are sent.').' '.$GLOBALS['I18N']->get('Reports will be sent by email to').' '.getConfig("report_address"));
      }
    }
! }
  
! Sql_query("SET SQL_BIG_TABLES=1");
! $script_stage = 2; # we know the messages to process
! include_once "footer.inc";
! if (!$num_per_batch) {
    $num_per_batch = 1000000;
! }
  
! while ($message = Sql_fetch_array($messages)) {
    $failed_sent = 0;
    $throttlecount = 0;
  
--- 339,354 ----
          output($GLOBALS['I18N']->get('Your webserver is running in safe_mode. Please keep this window open. It may reload several times to make sure all messages are sent.').' '.$GLOBALS['I18N']->get('Reports will be sent by email to').' '.getConfig("report_address"));
        }
      }
!   }
  
!   Sql_query("SET SQL_BIG_TABLES=1");
!   $script_stage = 2; # we know the messages to process
!   include_once "footer.inc";
!   if (!$num_per_batch) {
      $num_per_batch = 1000000;
!   }
  
!   while ($message = Sql_fetch_array($messages)) {
      $failed_sent = 0;
      $throttlecount = 0;
  
***************
*** 392,398 ****
        finish("info","Message $messageid: \nNo users apply for attributes, ie nothing to do");
        $script_stage = 6;
        # we should actually continue with the next message
!       return;
      }
    }
    if ($script_stage < 3)
--- 405,411 ----
          finish("info","Message $messageid: \nNo users apply for attributes, ie nothing to do");
          $script_stage = 6;
          # we should actually continue with the next message
!         continue;
        }
      }
      if ($script_stage < 3)
***************
*** 408,413 ****
--- 421,428 ----
      if (VERBOSE) {
        output($GLOBALS['I18N']->get('looking for users who can be excluded from this mailing'));
      }
+ 
+     /* Find out which users have ALREADY received message, so they can be excluded */
      $req = Sql_Query("select userid from {$tables["usermessage"]} where messageid = $messageid");
      $skipped = Sql_Affected_Rows();
      while ($row = Sql_Fetch_Row($req)) {
***************
*** 420,431 ****
    }
    # also exclude unconfirmed users, otherwise they'll block the process
    # will give quite different statistics than when used web based
! #  $req = Sql_Query("select id from {$tables["user"]} where !confirmed");
! #  while ($row = Sql_Fetch_Row($req)) {
! #    array_push($doneusers,$row[0]);
! #  }
    if (sizeof($doneusers))
!     $exclusion = " and listuser.userid not in (".join(",",$doneusers).")";
    if (USE_LIST_EXCLUDE) {
      $excluded_lists = Sql_Fetch_Row_Query(sprintf('select data from %s where name = "excludelist" and id = %d',
        $GLOBALS["tables"]["messagedata"],$messageid));
--- 435,446 ----
      }
      # also exclude unconfirmed users, otherwise they'll block the process
      # will give quite different statistics than when used web based
!   #  $req = Sql_Query("select id from {$tables["user"]} where !confirmed");
!   #  while ($row = Sql_Fetch_Row($req)) {
!   #    array_push($doneusers,$row[0]);
!   #  }
      if (sizeof($doneusers))
!       $exclusion = " and user.id not in (".join(",",$doneusers).")";
      if (USE_LIST_EXCLUDE) {
        $excluded_lists = Sql_Fetch_Row_Query(sprintf('select data from %s where name = "excludelist" and id = %d',
          $GLOBALS["tables"]["messagedata"],$messageid));
***************
*** 438,448 ****
        $query .= sprintf(' and listuser.listid not in (%s)',$excluded_lists[0]);
      }
      if (sizeof($skipusers))
!       $exclusion .= " and listuser.userid not in (".join(",",$skipusers).")";
    }
  
    $userconfirmed = ' and user.confirmed and !user.blacklisted ';
  
    $query = sprintf('select distinct user.id from
      %s as listuser,
      %s as user,
--- 453,476 ----
          $query .= sprintf(' and listuser.listid not in (%s)',$excluded_lists[0]);
        }
        if (sizeof($skipusers))
!         $exclusion .= " and user.id not in (".join(",",$skipusers).")";
      }
  
      $userconfirmed = ' and user.confirmed and !user.blacklisted ';
  
+     if ($pass==1)
+       $query = sprintf('select distinct user.id from
+         %s as user,
+         %s as usermessage_seq
+         where
+         usermessage_seq.messageid = %d and
+         user.id = usermessage_seq.userid %s %s',
+         $tables["user"],$tables['usermessage_seq'],
+         $messageid,
+         $userconfirmed,
+         $exclusion,
+         $user_attribute_query); 
+     else
        $query = sprintf('select distinct user.id from
          %s as listuser,
          %s as user,
***************
*** 533,540 ****
              # output("User matches message frequency");
              $rssitems = rssUserHasContent($userid,$messageid,$rssfrequency);
              $cansend = sizeof($rssitems) && (sizeof($rssitems) > $rss_content_treshold);
! #            if (!$cansend)
! #              output("No content to send for this user ".sizeof($rssitems));
            } else {
              $cansend = 0;
            }
--- 561,568 ----
                # output("User matches message frequency");
                $rssitems = rssUserHasContent($userid,$messageid,$rssfrequency);
                $cansend = sizeof($rssitems) && (sizeof($rssitems) > $rss_content_treshold);
!   #            if (!$cansend)
!   #              output("No content to send for this user ".sizeof($rssitems));
              } else {
                $cansend = 0;
              }
***************
*** 594,599 ****
--- 622,631 ----
              } else {
                $success = sendEmailTest($messageid,$useremail);
              }
+ 
+             /* Sent queued sequential e-mail, remove from queue */
+             if ($pass == 1)
+               Sql_query("delete * from {$tables['usermessage_seq']} where userid=$userid and messageid=$messageid");
              if ($success) {
                if (USE_DOMAIN_THROTTLE) {
                  list($mailbox,$domainname) = explode('@',$useremail);
***************
*** 672,680 ****
          # update possible other users matching this email as well,
          # to avoid duplicate sending when people have subscribed multiple times
          # bit of legacy code after making email unique in the database
! #        $emails = Sql_query("select * from {$tables['user']} where email =\"$useremail\"");
! #        while ($email = Sql_fetch_row($emails))
! #          Sql_query("replace into {$tables['usermessage']} (userid,messageid) values($email[0],$messageid)");
        }  else {
          # some "invalid emails" are entirely empty, ah, that is because they are unconfirmed
  
--- 704,712 ----
            # update possible other users matching this email as well,
            # to avoid duplicate sending when people have subscribed multiple times
            # bit of legacy code after making email unique in the database
!   #        $emails = Sql_query("select * from {$tables['user']} where email =\"$useremail\"");
!   #        while ($email = Sql_fetch_row($emails))
!   #          Sql_query("replace into {$tables['usermessage']} (userid,messageid) values($email[0],$messageid)");
          }  else {
            # some "invalid emails" are entirely empty, ah, that is because they are unconfirmed
  
***************
*** 754,763 ****
      if ($script_stage < 5)
        $script_stage = 5;
    }
! }
  
! if (!$num_messages)
    $script_stage = 6; # we are done
! # shutdown will take care of reporting
  
  ?>
--- 786,796 ----
        if ($script_stage < 5)
          $script_stage = 5;
      }
!   }
  
!   if (!$num_messages)
      $script_stage = 6; # we are done
!   # shutdown will take care of reporting
! }
  
  ?>