Discussion:
Perl, SWIG and multi threading
(too old to reply)
l***@hotmail.com
2006-08-03 11:52:29 UTC
Permalink
Hello,

I would very much appreciate your answers on the following issue.
I have a multi threaded application. Each thread is running a function
that creates a perl interpreter and uses it to parse and then run a
perl function from an arbitrary file.
After running this application for a while the application is stuck and

I can see that 10 threads are in the perl_parse function. An inner
looks show that all threads are in the SWIG_Init function created by
SWIG. I think it has something to do with the function
Swig_TypeClientData but i'm not sure.
I'm running on WInXP with perl 5.8.3.809, and I'm using SWIG 1.3.27 to
wrap my c++ classes and pass them to the perl functions.
Are any of you are familiar with such a problem? Do you have an idea
why this can happen?

I've attached my function code and marked the line where the threads
are "stuck":


__declspec(dllexport)
long CFTPerlUserExitInstanceRun(char *FileName,char *FuncName,struct
PerlParmInfo *ParmsVec,unsigned int ParmsVecSize,char* pReason,int
*pCount,int *pUserExitRc)
{
long lRc = 0;
int count = -1;
PerlInterpreter* my_perl = NULL;


char *embedding[] = { "", FileName};


while (true)
{
my_perl = perl_alloc();
if (my_perl == NULL)
{
lRc = -1;
break;
}


EnterCriticalSection(&g_ParserCS);
PERL_SET_CONTEXT(my_perl);
PL_perl_destruct_level = 1;
perl_construct(my_perl);
LeaveCriticalSection(&g_ParserCS);


// *****************************************************
// THREADS ARE STUCK IN PERL_PARSE
// *****************************************************
lRc = perl_parse(my_perl, xs_init, 2, embedding , NULL);


if (lRc != 0)
{
lRc = -2;
break;
}


dSP;
ENTER;
SAVETMPS;


PUSHMARK(SP);


// push parms to stack
char ParmType[MAX_PARM_TYPE_LEN];


for(unsigned int i=0; i<ParmsVecSize; i++)
{
strncpy(ParmType, PERL_WRAP_PACKAGE_NAME,
MAX_PARM_TYPE_LEN-1);
strncat(ParmType, "::", MAX_PARM_TYPE_LEN-3);
strncat(ParmType, ParmsVec[i].parmtype, MAX_PARM_TYPE_LEN-
strlen(PERL_WRAP_PACKAGE_NAME)-3);


SV *pSVParm = sv_newmortal();
sv_setref_pv( pSVParm, ParmType, ParmsVec[i].parm );
XPUSHs(pSVParm);
}


PUTBACK;
try
{
count = call_pv(FuncName, G_EVAL|G_SCALAR);
}
catch(...)
{
lRc = -3;
}


SPAGAIN;


// Check the eval first
if (SvTRUE(ERRSV))
{
STRLEN n_a;
strncpy(pReason, SvPV(ERRSV, n_a), MAX_PERL_REASON_LENGTH-1);
//@G1028
lRc = -4;
POPs ;
}
else
{
if (count != 1)
{
*pCount = count;
lRc = -5;
}
else
{
*pUserExitRc = POPi;


}
}


PUTBACK ;
FREETMPS ;
LEAVE ;


break;
}


if (my_perl != NULL)
{
EnterCriticalSection(&g_ParserCS);
PL_perl_destruct_level = 1;
perl_destruct(my_perl);
LeaveCriticalSection(&g_ParserCS);
perl_free(my_perl);
}


return lRc;



}


Thank you very much.
Lily.
Nick Ing-Simmons
2006-08-07 19:38:12 UTC
Permalink
Post by l***@hotmail.com
Hello,
I would very much appreciate your answers on the following issue.
I have a multi threaded application. Each thread is running a function
that creates a perl interpreter and uses it to parse and then run a
perl function from an arbitrary file.
After running this application for a while the application is stuck and
I can see that 10 threads are in the perl_parse function. An inner
looks show that all threads are in the SWIG_Init function created by
SWIG.
Hmm, can we see the code for SWIG_Init() - is _IT_ threadsafe
e.g. is SWIG using global variables so that SWIG_Init() only does
something once - if so there may be a race when you have multiple threads.
Post by l***@hotmail.com
I think it has something to do with the function
Swig_TypeClientData but i'm not sure.
I'm running on WInXP with perl 5.8.3.809, and I'm using SWIG 1.3.27 to
wrap my c++ classes and pass them to the perl functions.
Are any of you are familiar with such a problem? Do you have an idea
why this can happen?
I've attached my function code and marked the line where the threads
I can't see anything obviously wrong in code below.
Any chance you can post the fragment after CPP processing?
One theory I have is that #defines you are using to compile this code
don't match how your perl was compiled. So seeing how
PERL_SET_CONTEXT(my_perl) expands would allow me to guess how
USE_DECLSPEC_THREAD is being seen.


Or build with perl with debug so you can see where in perl_parse()
threads are "stuck".
Post by l***@hotmail.com
__declspec(dllexport)
long CFTPerlUserExitInstanceRun(char *FileName,char *FuncName,struct
PerlParmInfo *ParmsVec,unsigned int ParmsVecSize,char* pReason,int
*pCount,int *pUserExitRc)
{
long lRc = 0;
int count = -1;
PerlInterpreter* my_perl = NULL;
char *embedding[] = { "", FileName};
It is traditional to have a tailing NULL in an argv[] list
but I don't think perl cares.
Post by l***@hotmail.com
while (true)
{
my_perl = perl_alloc();
if (my_perl == NULL)
{
lRc = -1;
break;
}
EnterCriticalSection(&g_ParserCS);
PERL_SET_CONTEXT(my_perl);
PL_perl_destruct_level = 1;
perl_construct(my_perl);
LeaveCriticalSection(&g_ParserCS);
// *****************************************************
// THREADS ARE STUCK IN PERL_PARSE
// *****************************************************
lRc = perl_parse(my_perl, xs_init, 2, embedding , NULL);
if (lRc != 0)
{
lRc = -2;
break;
}
dSP;
ENTER;
SAVETMPS;
PUSHMARK(SP);
// push parms to stack
char ParmType[MAX_PARM_TYPE_LEN];
for(unsigned int i=0; i<ParmsVecSize; i++)
{
strncpy(ParmType, PERL_WRAP_PACKAGE_NAME,
MAX_PARM_TYPE_LEN-1);
strncat(ParmType, "::", MAX_PARM_TYPE_LEN-3);
strncat(ParmType, ParmsVec[i].parmtype, MAX_PARM_TYPE_LEN-
strlen(PERL_WRAP_PACKAGE_NAME)-3);
SV *pSVParm = sv_newmortal();
sv_setref_pv( pSVParm, ParmType, ParmsVec[i].parm );
XPUSHs(pSVParm);
}
PUTBACK;
try
{
count = call_pv(FuncName, G_EVAL|G_SCALAR);
}
catch(...)
{
lRc = -3;
}
SPAGAIN;
// Check the eval first
if (SvTRUE(ERRSV))
{
STRLEN n_a;
strncpy(pReason, SvPV(ERRSV, n_a), MAX_PERL_REASON_LENGTH-1);
lRc = -4;
POPs ;
}
else
{
if (count != 1)
{
*pCount = count;
lRc = -5;
}
else
{
*pUserExitRc = POPi;
}
}
PUTBACK ;
FREETMPS ;
LEAVE ;
break;
}
if (my_perl != NULL)
{
EnterCriticalSection(&g_ParserCS);
PL_perl_destruct_level = 1;
perl_destruct(my_perl);
LeaveCriticalSection(&g_ParserCS);
perl_free(my_perl);
}
return lRc;
}
Thank you very much.
Lily.
l***@hotmail.com
2006-08-08 14:52:52 UTC
Permalink
Hi again,

I think you might have a point there - the SWIG code might not be
thread safe.
found the following in SWIGs documentation (last paragraph is the
important one):

15.1 The SWIG runtime code
Many of SWIG's target languages generate a set of functions commonly
known as the "SWIG runtime." These functions are primarily related to
the runtime type system which checks pointer types and performs other
tasks such as proper casting of pointer values in C++. As a general
rule, the statically typed target languages, such as Java, use the
language's built in static type checking and have no need for a SWIG
runtime. All the dynamically typed / interpreted languages rely on the
SWIG runtime.

The runtime functions are private to each SWIG-generated module. That
is, the runtime functions are declared with "static" linkage and are
visible only to the wrapper functions defined in that module. The only
problem with this approach is that when more than one SWIG module is
used in the same application, those modules often need to share type
information. This is especially true for C++ programs where SWIG must
collect and share information about inheritance relationships that
cross module boundaries.

To solve the problem of sharing information across modules, a pointer
to the type information is stored in a global variable in the target
language namespace. During module initialization, type information is
loaded into the global data structure of type information from all
modules.

This can present a problem with threads. If two modules try and load at
the same time, the type information can become corrupt. SWIG currently
does not provide any locking, and if you use threads, you must make
sure that modules are loaded serially. Be careful if you use threads
and the automatic module loading that some scripting languages provide.
One solution is to load all modules before spawning any threads.
---
Looks like using this code from multiple threads can casue my problem,
but i would expect an exception or wierd behaviour in this case,
instead of endless loop / deadlock.
I still have to confirm that this is my problem,
Thanks again,
Lily.
Post by Nick Ing-Simmons
Post by l***@hotmail.com
Hello,
I would very much appreciate your answers on the following issue.
I have a multi threaded application. Each thread is running a function
that creates a perl interpreter and uses it to parse and then run a
perl function from an arbitrary file.
After running this application for a while the application is stuck and
I can see that 10 threads are in the perl_parse function. An inner
looks show that all threads are in the SWIG_Init function created by
SWIG.
Hmm, can we see the code for SWIG_Init() - is _IT_ threadsafe
e.g. is SWIG using global variables so that SWIG_Init() only does
something once - if so there may be a race when you have multiple threads.
Post by l***@hotmail.com
I think it has something to do with the function
Swig_TypeClientData but i'm not sure.
I'm running on WInXP with perl 5.8.3.809, and I'm using SWIG 1.3.27 to
wrap my c++ classes and pass them to the perl functions.
Are any of you are familiar with such a problem? Do you have an idea
why this can happen?
I've attached my function code and marked the line where the threads
I can't see anything obviously wrong in code below.
Any chance you can post the fragment after CPP processing?
One theory I have is that #defines you are using to compile this code
don't match how your perl was compiled. So seeing how
PERL_SET_CONTEXT(my_perl) expands would allow me to guess how
USE_DECLSPEC_THREAD is being seen.
Or build with perl with debug so you can see where in perl_parse()
threads are "stuck".
Post by l***@hotmail.com
__declspec(dllexport)
long CFTPerlUserExitInstanceRun(char *FileName,char *FuncName,struct
PerlParmInfo *ParmsVec,unsigned int ParmsVecSize,char* pReason,int
*pCount,int *pUserExitRc)
{
long lRc = 0;
int count = -1;
PerlInterpreter* my_perl = NULL;
char *embedding[] = { "", FileName};
It is traditional to have a tailing NULL in an argv[] list
but I don't think perl cares.
Post by l***@hotmail.com
while (true)
{
my_perl = perl_alloc();
if (my_perl == NULL)
{
lRc = -1;
break;
}
EnterCriticalSection(&g_ParserCS);
PERL_SET_CONTEXT(my_perl);
PL_perl_destruct_level = 1;
perl_construct(my_perl);
LeaveCriticalSection(&g_ParserCS);
// *****************************************************
// THREADS ARE STUCK IN PERL_PARSE
// *****************************************************
lRc = perl_parse(my_perl, xs_init, 2, embedding , NULL);
if (lRc != 0)
{
lRc = -2;
break;
}
dSP;
ENTER;
SAVETMPS;
PUSHMARK(SP);
// push parms to stack
char ParmType[MAX_PARM_TYPE_LEN];
for(unsigned int i=0; i<ParmsVecSize; i++)
{
strncpy(ParmType, PERL_WRAP_PACKAGE_NAME,
MAX_PARM_TYPE_LEN-1);
strncat(ParmType, "::", MAX_PARM_TYPE_LEN-3);
strncat(ParmType, ParmsVec[i].parmtype, MAX_PARM_TYPE_LEN-
strlen(PERL_WRAP_PACKAGE_NAME)-3);
SV *pSVParm = sv_newmortal();
sv_setref_pv( pSVParm, ParmType, ParmsVec[i].parm );
XPUSHs(pSVParm);
}
PUTBACK;
try
{
count = call_pv(FuncName, G_EVAL|G_SCALAR);
}
catch(...)
{
lRc = -3;
}
SPAGAIN;
// Check the eval first
if (SvTRUE(ERRSV))
{
STRLEN n_a;
strncpy(pReason, SvPV(ERRSV, n_a), MAX_PERL_REASON_LENGTH-1);
lRc = -4;
POPs ;
}
else
{
if (count != 1)
{
*pCount = count;
lRc = -5;
}
else
{
*pUserExitRc = POPi;
}
}
PUTBACK ;
FREETMPS ;
LEAVE ;
break;
}
if (my_perl != NULL)
{
EnterCriticalSection(&g_ParserCS);
PL_perl_destruct_level = 1;
perl_destruct(my_perl);
LeaveCriticalSection(&g_ParserCS);
perl_free(my_perl);
}
return lRc;
}
Thank you very much.
Lily.
Loading...