Author: gnichols
Date: 2010-09-02 16:02:50 +0000 (Thu, 02 Sep 2010)
New Revision: 669
Modified:
trunk/tests/memory/threaded_memtest.c
Log:
629082 - threaded_memtest hangs on very large systems.
Modified: trunk/tests/memory/threaded_memtest.c
===================================================================
--- trunk/tests/memory/threaded_memtest.c 2010-09-02 16:00:04 UTC (rev 668)
+++ trunk/tests/memory/threaded_memtest.c 2010-09-02 16:02:50 UTC (rev 669)
@@ -80,17 +80,25 @@
pthread_cond_t test_start = PTHREAD_COND_INITIALIZER;
pthread_mutex_t finish_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t finish_cond = PTHREAD_COND_INITIALIZER;
+pthread_mutex_t running_threads_mutex = PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t running_threads_cond = PTHREAD_COND_INITIALIZER;
unsigned done = 0;
-unsigned running_threads = 0;
+long unsigned running_threads = 0;
/* short name of the program */
char *basename = NULL;
/* set the affinity for the current task to the given CPU */
+/* This now uses dynamic cpu_sets as the convention cpu_set_t
+ was limited to 1024p */
int on_cpu(unsigned cpu){
- cpu_set_t mask;
- CPU_ZERO(&mask);
- CPU_SET(cpu,&mask);
- if (setaffinity(mask) < 0){
+ cpu_set_t* mask;
+ size_t masksize;
+ mask = CPU_ALLOC(num_cpus);
+ masksize = CPU_ALLOC_SIZE(num_cpus);
+
+ CPU_ZERO_S(masksize, mask);
+ CPU_SET_S(cpu, masksize, mask);
+ if (sched_setaffinity(0, masksize, mask) < 0) {
perror("sched_setaffinity");
return -1;
}
@@ -181,30 +189,41 @@
lp[1]=thread_id;
lp[2]=i;
}
- /* Okay, we have grabbed our memory - this thread is now live */
- pthread_mutex_lock(<_mutex);
- live_threads++;
- pthread_mutex_unlock(<_mutex);
+ /* Okay, we have grabbed our memory */
if (verbose) printf("thread %ld: mapping complete\n",thread_id);
/* let main() go now that the thread is finished initializing. */
if (!parallel) {
mmap_done=1;
pthread_cond_signal(&mmap_cond);
- } else if (live_threads == num_threads) {
+ pthread_mutex_unlock(&mmap_mutex);
+ }
+
+ /*
+ * Incrementing live_threads inside test_mutex avoids a timing
+ * sensitive race -- otherwise some threads could miss the
+ * pthread_cond_broadcast of test_start.
+ */
+ pthread_mutex_lock(&test_mutex);
+ live_threads++;
+ if (live_threads == num_threads) {
/* if this is the last thread to init, let main() know we're done */
+ /* NOTE: only the last thread sends this signal */
pthread_cond_signal(&init_cond);
}
/* Wait for the signal to begin testing */
- pthread_mutex_lock(&test_mutex);
while (start.tv_sec == 0) {
pthread_cond_wait(&test_start,&test_mutex);
}
+ pthread_mutex_unlock(&test_mutex);
+
+ pthread_mutex_lock(&running_threads_mutex);
running_threads++;
- pthread_mutex_unlock(&test_mutex);
- if (verbose) printf("thread %lu: test start\n",thread_id);
-
+ if (verbose)
+ printf("thread %lu (%lu): test start\n",thread_id,running_threads);
+ pthread_mutex_unlock(&running_threads_mutex);
+
loop_counters[thread_id]=0;
while (!done) {
/* Choose a random thread and a random page */
@@ -229,14 +248,21 @@
loop_counters[thread_id]++;
}
+
+
/* make sure everyone's finished before we unmap */
pthread_mutex_lock(&finish_mutex);
- if (verbose) printf("thread %lu finished.\n",thread_id);
+
running_threads--;
- if (running_threads==0)
- pthread_cond_broadcast(&finish_cond);
- else
- while (running_threads) { pthread_cond_wait(&finish_cond,&finish_mutex); }
+ if (verbose)
+ printf("thread %lu (%lu): test start\n",thread_id,running_threads);
+ if (running_threads == 0)
+ pthread_cond_broadcast(&finish_cond); /* This is the cleanup thread */
+ else {
+ while (running_threads > 0) {
+ pthread_cond_wait(&finish_cond,&finish_mutex);
+ }
+ }
pthread_mutex_unlock(&finish_mutex);
/* Clean up and exit. */
@@ -356,6 +382,10 @@
mmap_regions=(char **)malloc(num_threads*sizeof(char *));
loop_counters=(unsigned long *)malloc(num_threads*sizeof(unsigned long *));
+ for (i = 0; i < num_threads; i++) {
+ mmap_regions[i] = NULL;
+ loop_counters[i] = 0;
+ }
/* Create all our threads! */
while (created_threads < num_threads) {
pthread_mutex_lock(&mmap_mutex);
@@ -374,10 +404,12 @@
if (parallel) {
/* Wait for the signal that everyone is finished initializing */
pthread_mutex_lock(&init_mutex);
- while (live_threads < num_threads) { pthread_cond_wait(&init_cond,&init_mutex); }
+ while (live_threads < num_threads) {
+ pthread_cond_wait(&init_cond,&init_mutex);
+ }
pthread_mutex_unlock(&init_mutex);
}
-
+
/* Let the testing begin! */
if (!verbose && !quiet) printf("\n");
gettimeofday(&start,NULL);
@@ -389,6 +421,13 @@
mysig.sa_flags=0;
sigaction(SIGINT,&mysig,NULL);
+ /* Wait until all threads are actually running otherwise some threads
+ never get started before done is set on large UV systems */
+
+ while (running_threads < num_threads) {
+ if (!quiet) progressbar("Running Threads", running_threads, num_threads);
+ sleep((num_threads - running_threads) / 60 + 1);
+ }
/* Wait for the allotted time */
i=0;
while (!done && (i<runtime)) {
@@ -403,6 +442,7 @@
while (live_threads) {
pthread_join(threads[live_threads-1],NULL);
live_threads--;
+ if (!quiet) progressbar("Joined Threads", (num_threads - live_threads), num_threads);
}
gettimeofday(&finish,NULL);
if (!quiet) printf("\n");