| 1 |
=============================
|
| 2 |
|
| 3 |
Installation
|
| 4 |
------------
|
| 5 |
|
| 6 |
1) Install Drupal, OG, OG2list, OG_forum. Don't set up any automatic
|
| 7 |
cronjobs.
|
| 8 |
|
| 9 |
2) Set up some groups.
|
| 10 |
|
| 11 |
3) Create dummy users using generate-users.php. http://cvs.drupal.org/viewcvs/drupal/contributions/modules/devel/generate/ Use a subdomain as a mail domain that you have full control over. We want to count incoming mails.
|
| 12 |
|
| 13 |
4) Assing dummy users to groups using generate-og-users.php
|
| 14 |
|
| 15 |
5) You can create dummy incoming mails using generate-og2list-mail.php
|
| 16 |
if you don't want to send real mails. The incoming mails can be sent
|
| 17 |
out again through mail-out.pl.
|
| 18 |
This is in the generate sub directory of the DEVEL module from CVS HEAD.
|
| 19 |
To use the script, browse to it in your browser. You should see a blank page.
|
| 20 |
mfa@punlc.local:~$ lynx --source john.localdomain/drupal/generate-og2list-mail.php
|
| 21 |
The amount of mail generated is dependent upon the number of users attached to groups.
|
| 22 |
|
| 23 |
We wrapped the create_content function with a for loop to be able to control the amount of mail being created each run.
|
| 24 |
|
| 25 |
function gom_create_content($groups) {
|
| 26 |
for ($i = 1; $i <= 50; $i++) {
|
| 27 |
foreach ($groups as $nid => $group) {
|
| 28 |
foreach ($group['users'] as $user) {
|
| 29 |
$msg_id = '<'. time() .'.'. mt_rand() .'@'. strtolower(variable_get('og2list_domain', $_SERVER['SERVER_NAME'])) .'>';
|
| 30 |
|
| 31 |
$subject = 'Test mail from '. $user['name'] .' to '. $group['recipient'];
|
| 32 |
db_query("INSERT INTO {og2list_incoming_content} (from_address,from_name,subject,msgid,content_type,body) VALUES ('%s', '%s', '%s', '%s', '%s', '%s')", $user['mail'], $user['name'], $subject, $msg_id, 'text/plain', "totally irrel$
|
| 33 |
db_query("INSERT INTO {og2list_incoming_groups} SET mid=(SELECT mid FROM {og2list_incoming_content} WHERE msgid='%s'), oid=(SELECT nid FROM {og2list_groups} WHERE recipient='%s')", $msg_id, $group['recipient']);
|
| 34 |
}
|
| 35 |
}
|
| 36 |
}
|
| 37 |
}
|
| 38 |
|
| 39 |
|
| 40 |
6) Count how many mails are in the queue by counting the rows on og2list_incoming_content
|
| 41 |
select * from og2list_incoming_content
|
| 42 |
7) Put some calls to time() into cron.php and og2list.module to measure 2 things
|
| 43 |
|
| 44 |
1: How long cron takes to process
|
| 45 |
2: Specifically how much time is og2list taking to process mail.
|
| 46 |
|
| 47 |
Which in turn tells us some other interesting things
|
| 48 |
|
| 49 |
1: What percentage of the time cron takes is being used by og2list.
|
| 50 |
2: The exact amount of time cron takes to send a certain number of mails.
|
| 51 |
|
| 52 |
8) Run cron
|
| 53 |
mfa@punlc.local:~$ lynx --source john.localdomain/drupal/cron.php
|
| 54 |
|
| 55 |
We implemented some simple stat tracking code into og2list.module and cron.php
|
| 56 |
Here is an example of what cron.php could look like and og2list_cron() should look like during testing.
|
| 57 |
You can also tune the amount of mail that og2list sends in a single batch by changing the default in variable_get('og2list_incoming_number', 10)
|
| 58 |
|
| 59 |
|
| 60 |
EXAMPLE CRON.PHP
|
| 61 |
|
| 62 |
// Iterate through the modules calling their cron handlers (if any):
|
| 63 |
$time_begin = time();
|
| 64 |
module_invoke_all('cron');
|
| 65 |
|
| 66 |
// Clean up
|
| 67 |
variable_set('cron_busy', false);
|
| 68 |
$time_end = time();
|
| 69 |
$time_diff = $time_end - $time_begin;
|
| 70 |
watchdog('cron', 'cron took so long: '. $time_diff);
|
| 71 |
variable_set('cron_last', time());
|
| 72 |
watchdog('cron', t('Cron run completed'));
|
| 73 |
|
| 74 |
EXAMPLE OGLIST.MODULE - og2list_cron
|
| 75 |
|
| 76 |
function og2list_cron() {
|
| 77 |
$counter = 0;
|
| 78 |
$time_begin = time();
|
| 79 |
$lockfile = og2list_lockfile();
|
| 80 |
// obtain lock to prevent corruption
|
| 81 |
$fp = @fopen(file_create_path($lockfile), 'w');
|
| 82 |
if (flock($fp, LOCK_EX)) {
|
| 83 |
// initialize random number generator
|
| 84 |
mt_srand((double)microtime()*1000000);
|
| 85 |
// store the original cron user
|
| 86 |
og2list_switch_user();
|
| 87 |
$result = db_query_range('SELECT c.* FROM {og2list_incoming_groups} g INNER JOIN {og2list_incoming_content} c ON g.mid = c.mid WHERE g.oid != 0 ORDER BY c.mid ASC', 0, variable_get('og2list_incoming_number', 10));
|
| 88 |
while ($mail = db_fetch_object($result)) {
|
| 89 |
$counter++;
|
| 90 |
$result2 = db_query('SELECT oid FROM {og2list_incoming_groups} WHERE mid = %d', $mail->mid);
|
| 91 |
$mail->og_nids = array();
|
| 92 |
|
| 93 |
while ($group_id = db_fetch_object($result2)) {
|
| 94 |
$mail->og_nids[] = $group_id->oid;
|
| 95 |
}
|
| 96 |
if (count($mail->og_nids)) {
|
| 97 |
$status = og2list_process_mail($mail);
|
| 98 |
watchdog('debug', serialize($status));
|
| 99 |
if ($status) {
|
| 100 |
db_query('DELETE FROM {og2list_incoming_content} WHERE mid = %d', $mail->mid);
|
| 101 |
db_query('DELETE FROM {og2list_incoming_groups} WHERE mid = %d', $mail->mid);
|
| 102 |
}
|
| 103 |
}
|
| 104 |
}
|
| 105 |
if (module_exist('mailcommand')) {
|
| 106 |
$result = db_query_range('SELECT c.* FROM {og2list_incoming_groups} g INNER JOIN {og2list_incoming_content} c ON g.mid = c.mid WHERE g.oid = 0 ORDER BY c.mid ASC', 0, 10);
|
| 107 |
while ($mail = db_fetch_object($result)) {
|
| 108 |
$status = mailcommand_process_mail($mail);
|
| 109 |
if ($status) {
|
| 110 |
db_query('DELETE FROM {og2list_incoming_content} WHERE mid = %d', $mail->mid);
|
| 111 |
db_query('DELETE FROM {og2list_incoming_groups} WHERE mid = %d', $mail->mid);
|
| 112 |
}
|
| 113 |
}
|
| 114 |
}
|
| 115 |
// revert to the original cron user
|
| 116 |
og2list_switch_user();
|
| 117 |
$time_end = time();
|
| 118 |
$time_diff = $time_end - $time_begin;
|
| 119 |
watchdog('og2list', 'og2list process this many messages: ' . $counter);
|
| 120 |
watchdog('og2list', 'og2list took this long: ' . $time_diff);
|
| 121 |
og2list_unlock($fp);
|
| 122 |
}
|
| 123 |
else {
|
| 124 |
watchdog('og2list', t('Could not obtain og2list lock.'), WATCHDOG_ERROR);
|
| 125 |
}
|
| 126 |
}
|
| 127 |
|
| 128 |
Testing scenarios
|
| 129 |
-----------------
|
| 130 |
|
| 131 |
1) Processing incoming mail
|
| 132 |
|
| 133 |
Either send a lot of mails to the test domain or generate them
|
| 134 |
through generate-og2list-mail.php.
|
| 135 |
|
| 136 |
Run cron.php: This will trigger 10 mails to be taken from the incoming
|
| 137 |
queue and to be processed. Stop the time this needs.
|
| 138 |
|
| 139 |
Increase the variable og2list_incoming_number in settings.php and try
|
| 140 |
again.
|
| 141 |
|
| 142 |
Increase the number until cron.php runs maybe 2 minutes (Drupal
|
| 143 |
increases the timeout limit to 4 minutes, we need to account for other
|
| 144 |
processes such as the search indexer). Infact you could try this with
|
| 145 |
and without search.module enabled.
|
| 146 |
|
| 147 |
The number you got is the maximal number of mails you can process
|
| 148 |
during one cron run. Now if you run a cronjob every 5 minutes, then
|
| 149 |
the amount of received mails per hour should be 12 times your number.
|
| 150 |
|
| 151 |
If this number is not high enough (I expect a few hundred mails at
|
| 152 |
least on decent hardware) then we need to think again.
|
| 153 |
|
| 154 |
1 a) It will be interesting to find out how this incoming mails per
|
| 155 |
hour value correlates with the number of users in a group. I have
|
| 156 |
written og2list.module in such a way as to be affected in the least
|
| 157 |
possible way by this number. Would be nice to know I did it right.
|
| 158 |
|
| 159 |
To test this create only one group and enable more and more users in
|
| 160 |
it. See how the time that cron.php needs for a fixed number of mail
|
| 161 |
changes. Ideally it will change very little.
|
| 162 |
|
| 163 |
For all tests it is mandatory that you ensure that you still got
|
| 164 |
enough mails in the queue.
|
| 165 |
|
| 166 |
1) Injecting Mail Directly into the Database
|
| 167 |
|
| 168 |
Make sure the recipient and sender in this command are correct.
|
| 169 |
The sending address needs to be connected to the og2list group you are trying to send to and must have permission to post.
|
| 170 |
The recipient must be a valid group address. To verify your groups address you must look in the og2list_groups table.
|
| 171 |
|
| 172 |
There is a testmails directory in the OG2list folder. Execute this command as the og2list user
|
| 173 |
|
| 174 |
cd /home/og2list/
|
| 175 |
./mail-in.pl -f from@avalidsender.com -t valid@groupaddress.com < /path/to/testmails/killes/tb-arabic
|
| 176 |
|
| 177 |
|
| 178 |
2) Sending mail
|
| 179 |
|
| 180 |
When mail is received the copies that go out to the users are
|
| 181 |
generated and stored in the database at the same time. This means we
|
| 182 |
already accounted for this in step 1.
|
| 183 |
|
| 184 |
The actual mailing is done by mail-out.pl. It is run from a cronjob.
|
| 185 |
|
| 186 |
After being called mail-out.pl will try to send out all mails that are
|
| 187 |
in the queue by feeding them one after another to the MTA.
|
| 188 |
|
| 189 |
Now create a test group with yourself as a member and a fixed, big
|
| 190 |
number of dummy users. Disable most of the dummy users by setting
|
| 191 |
their is_active bit to 0 in the og_uid table. Send one mail to the
|
| 192 |
mail alias and run cron.php. The mails for the dummy users get
|
| 193 |
prepared. Now run mail-out.pl from the commandline. Log the system
|
| 194 |
load and take the time until mail-in.pl has finished. On the receiving
|
| 195 |
MTA, log the timestamps for the incoming mails to the dummy users.
|
| 196 |
|
| 197 |
Repeat with more users set to is_active = 1 in the test group until
|
| 198 |
you think you've had enough. Draw graphs from results.
|
| 199 |
|
| 200 |
|
| 201 |
This will put mail in the incoming_content table.
|
| 202 |
mfa@punlc.local:~$ lynx --source http://john.localdomain/drupal/generate-og2list-mail.php
|
| 203 |
|
| 204 |
This will take the mail from incoming_content and place it in the outgoing_content queue.
|
| 205 |
mfa@punlc.local:~$ lynx --source http://john.localdomain/drupal/cron.php
|
| 206 |
|
| 207 |
This processes the queue.
|
| 208 |
sudo su og2list; /Users/og2list/./mail-out.pl
|
| 209 |
|
Parent Directory
| 
Revision Log
| 
Revision Graph