www.digitalmars.com         C & C++   DMDScript  

digitalmars.D.learn - segfault on gc.impl.conservative.gc.Gcx.smallAlloc

reply Mengu <mengukagan gmail.com> writes:
hi all

this following code block [0] is exiting with "terminated by 
signal SIGBUS (Misaligned address error)" error. it processes 
like 200K rows and then fails. any ideas?

void getHotels() {
   import std.parallelism : taskPool;
   import std.functional  : partial;

   auto sunHotels         = new SunHotels();
   auto destinations      = sunHotels.parseDestinations();
   auto conn              = client.lockConnection();
   auto destinationResult = conn.execStatement("SELECT 
provider_ref, id FROM hotels_destination", ValueFormat.BINARY);
   int[int] destinationIds;

   foreach (row; rangify(destinationResult)) {
     destinationIds[row["provider_ref"].as!PGinteger] = 
row["id"].as!PGinteger;
   }

   foreach (destination; parallel(destinations)) {
     const string destId = to!string(destination.destinationId);
     const auto destinationFilePath = getcwd() ~ 
"/ext/data/hotels/" ~ destId ~ ".xml";
     auto xmlFile = new MmFile(destinationFilePath);
     auto fileContents = cast(string)xmlFile[0..xmlFile.length];
     auto hotels = sunHotels.parseHotelsResult(fileContents);
     const destIdInDb = destinationIds[destination.destinationId];
     auto sqls = appender!string;
     writeln("parsing destination: ", destination.destinationName);

     foreach (hotel; parallel(hotels)) {
         sqls.put(hotel.generateSql(destIdInDb).data.join);
     }

     sqls.data.writeln;


     delete fileContents;
     delete xmlFile;
   }

}

this is the full trace i got [1]:

#0  0x0000000000bef5ef in 
gc.impl.conservative.gc.Gcx.smallAlloc(ubyte, ref ulong, uint) ()
[Current thread is 1 (LWP 100171)]
(gdb) bt full
#0  0x0000000000bef5ef in 
gc.impl.conservative.gc.Gcx.smallAlloc(ubyte, ref ulong, uint) ()
No symbol table info available.
#1  0x0000000000bf3925 in 
gc.impl.conservative.gc.ConservativeGC.runLocked!(gc.impl.conservative.gc.ConservativeGC
mallocNoSync(ulong, uint, ref ulong, const(TypeInfo)),
gc.impl.conservative.gc.mallocTime, gc.impl.conservative.gc.numMallocs, ulong,
uint, ulong, const(TypeInfo)).runLocked(ref ulong, ref uint, ref ulong, ref
const(TypeInfo)) ()
No symbol table info available.
#2  0x0000000000bed103 in 
gc.impl.conservative.gc.ConservativeGC.qalloc(ulong, uint, 
const(TypeInfo)) ()
No symbol table info available.
#3  0x0000000000b9c6e3 in gc_qalloc ()
No symbol table info available.
#4  0x0000000000b96140 in core.memory.GC.qalloc(ulong, uint, 
const(TypeInfo)) ()
No symbol table info available.
#5  0x000000000093fdce in 
std.array.Appender!(immutable(char)[]).Appender.ensureAddable(ulong) (this=...,
nelems=761)
     at 
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:2929
         len = 394
         reqlen = 1155
         newlen = 1155
         u = 0
         overflow = false
         nbytes = 1155
         bi = {base = 0x2b1, size = 140737488337376, attr = 
4294949280}
#6  0x0000000000942c0f in 
std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).put(immutable(char)[
).bigDataFun(ulong) (
     this=0x7fffffffb9f0, extra=761)
     at 
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3023
No locals.
#7  0x0000000000942b55 in 
std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).pu
(immutable(char)[]) (this=...,
---Type <return> to continue, or q <return> to quit---
      at 
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3026
         bigData = "'"
         len = 760
         newlen = 34799909888
#8  0x000000000093e80a in hotel.Hotel.generateSql(int) (this=..., 
destinationId=5743) at source/hotel.d:216
         sqls = {_data = 0x81d5ca6c0}
         sql = {_data = 0x81d5ca6e0}
         childSqls = {_data = 0x81d5ca700}
#9  0x00000000009194c9 in app.getHotels().__foreachbody1(ref 
destination.Destination).__foreachbody2(ref hotel.Hotel) 
(this=0x7fffffffd2a0,
     __applyArg0=...) at source/app.d:211
         hotel = {provider_ref = 121475, destinationId = 7931, 
resortId = 11313, transfer = 0, roomTypes = {{rooms = {{roomType 
= 0x81a351188,
                       hotelId = 121475, roomId = 5369802, beds = 
2, extrabeds = 0, meals = 0x0}}, roomType = "Twin/Double room",
                 roomTypeId = 21}, {rooms = {{roomType = 
0x81a35c108, hotelId = 121475, roomId = 5761375, beds = 2, 
extrabeds = 0,
                       meals = 0x0}}, roomType = "Double room 
Queen bed", roomTypeId = 2651}}, reviews = 0x0, distance = 
nan(0xc000000000000),
           type = "Hotel", name = "Best Western Carriage House Inn 
and Suites", addr_1 = "1936 Highway 45 Bypass", addr_2 = 0x0,
           zip_code = "38305", city = "Jackson", state = "TN", 
country = "United States", country_code = "US",
           address = "1936 Highway 45 Bypass  38305 Jackson TN 
United States",
           mapurl = 
"http://www.sunhotels.net/GoogleMaps/showGoogleMap.asp?hotelId=121475&cc=en",
           headline = "With a stay at Best Western Carriage House 
Inn & Suites in Jackson, you'll be minutes from Casey Jones 
Village and close to Old Hickory Mall",
           description = "With a stay at Best Western Carriage 
House Inn & Suites in Jackson, you'll be minutes from Casey Jones 
Village and close to Old Hickory Mall.  This hotel is within the 
vicinity of International Rock-A"..., resort = "Jackson", 
destination = "Jackson (TN)",
           images = {{hotelId = 121475, id = 2378342}, {hotelId = 
121475, id = 2378343}, {hotelId = 121475, id = 2378344}, {hotelId 
= 121475,
                 id = 2378345}, {hotelId = 121475, id = 2378347}, 
{hotelId = 121475, id = 2378348}, {hotelId = 121475, id = 
2378349}, {
                 hotelId = 121475, id = 2378350}, {hotelId = 
121475, id = 2378351}, {hotelId = 121475, id = 2378352}, {hotelId 
= 121475,
                 id = 2378353}, {hotelId = 121475, id = 2378354}, 
{hotelId = 121475, id = 2378355}, {hotelId = 121475, id = 
2378356}, {
                 hotelId = 121475, id = 2378357}, {hotelId = 
121475, id = 2378359}, {hotelId = 121475, id = 2378360}, {hotelId 
= 121475,
                 id = 2378362}, {hotelId = 121475, id = 2378363}, 
{hotelId = 121475, id = 2378364}, {hotelId = 121475, id = 
2378365}, {
                 hotelId = 121475, id = 2378366}, {hotelId = 
121475, id = 2378368}, {hotelId = 121475, id = 2378369}, {hotelId 
= 121475,
                 id = 2378371}, {hotelId = 121475, id = 2378379}, 
{hotelId = 121475, id = 2378386}, {hotelId = 121475, id = 
2378389}, {
                 hotelId = 121475, id = 2378395}}, classification 
= "2", features = 0x0, timezone = "-06:00", latitude = 
"35.660888671875",
           longitude = "-88.853157043457031"}
#10 0x0000000000925c54 in 
std.parallelism.ParallelForeach!(hotel.Hotel[]).ParallelForeach.opApply(scope
int(ref hotel.Hotel) delegate).doIt() (
     this=0x7fffffffd0d0) at 
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/parallelism.d-mixin-3824:3870
         myUnitIndex = 3
         start = 3
         end = 4
---Type <return> to continue, or q <return> to quit---
         __key6543 = 3
         __limit6544 = 4
         i = 3
         __EAX = 0x81d5d4800
         __EDX = 1730176101
         __handler = 7
         __exception_object = 0x6
#11 0x0000000000bbe328 in std.parallelism.run!(void() 
delegate).run(void() delegate) ()
No symbol table info available.
#12 0x0000000000bbde88 in 
std.parallelism.Task!(std.parallelism.run, void() 
delegate).Task.impl(void*) ()
No symbol table info available.
#13 0x0000000000c121fb in std.parallelism.AbstractTask.job() ()
No symbol table info available.
#14 0x0000000000bbdb60 in 
std.parallelism.submitAndExecute(std.parallelism.TaskPool, scope 
void() delegate) ()
No symbol table info available.
#15 0x0000000000925b5a in 
std.parallelism.ParallelForeach!(hotel.Hotel[]).ParallelForeach.opApply(scope
int(ref hotel.Hotel) delegate) (
     this=..., dg=...) at 
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/parallelism.d-mixin-3824:3876
         workUnitIndex = 5
         len = 26
         shouldContinue = true

[0] https://pastebin.com/JmsTKU95
[1] https://pastebin.com/v1Yufz7g
Sep 25
next sibling parent reply Vladimir Panteleev <thecybershadow.lists gmail.com> writes:
On Monday, 25 September 2017 at 21:34:40 UTC, Mengu wrote:
     delete fileContents;
This looks suspicious - it is a slice of the memory-mapped file, not memory on the GC-managed heap, so "delete" is inapplicable to it. The GC ought to throw an exception when attempting to delete things not on the GC heap though. I think the stack trace itself looks like something that should only happen when the GC's internal data structures are corrupted, so you may want to investigate in that direction.
Sep 25
parent Mengu <mengukagan gmail.com> writes:
On Tuesday, 26 September 2017 at 00:36:36 UTC, Vladimir Panteleev 
wrote:
 On Monday, 25 September 2017 at 21:34:40 UTC, Mengu wrote:
     delete fileContents;
This looks suspicious - it is a slice of the memory-mapped file, not memory on the GC-managed heap, so "delete" is inapplicable to it. The GC ought to throw an exception when attempting to delete things not on the GC heap though. I think the stack trace itself looks like something that should only happen when the GC's internal data structures are corrupted, so you may want to investigate in that direction.
thanks vladimir, i'll look into that. i also should mention that everything works as expected on mac os x.
Sep 26
prev sibling next sibling parent Mengu <mengukagan gmail.com> writes:
On Monday, 25 September 2017 at 21:34:40 UTC, Mengu wrote:
 hi all

 this following code block [0] is exiting with "terminated by 
 signal SIGBUS (Misaligned address error)" error. it processes 
 like 200K rows and then fails. any ideas?

 [...]
hi all does anyone else have any ideas?
Sep 26
prev sibling parent reply drug <drug2004 bk.ru> writes:
26.09.2017 00:34, Mengu пишет:
 hi all
 
 this following code block [0] is exiting with "terminated by signal 
 SIGBUS (Misaligned address error)" error. it processes like 200K rows 
 and then fails. any ideas?
 
 void getHotels() {
    import std.parallelism : taskPool;
    import std.functional  : partial;
 
    auto sunHotels         = new SunHotels();
    auto destinations      = sunHotels.parseDestinations();
    auto conn              = client.lockConnection();
    auto destinationResult = conn.execStatement("SELECT provider_ref, id 
 FROM hotels_destination", ValueFormat.BINARY);
    int[int] destinationIds;
 
    foreach (row; rangify(destinationResult)) {
      destinationIds[row["provider_ref"].as!PGinteger] = 
 row["id"].as!PGinteger;
    }
 
    foreach (destination; parallel(destinations)) {
      const string destId = to!string(destination.destinationId);
      const auto destinationFilePath = getcwd() ~ "/ext/data/hotels/" ~ 
 destId ~ ".xml";
      auto xmlFile = new MmFile(destinationFilePath);
      auto fileContents = cast(string)xmlFile[0..xmlFile.length];
      auto hotels = sunHotels.parseHotelsResult(fileContents);
      const destIdInDb = destinationIds[destination.destinationId];
      auto sqls = appender!string;
      writeln("parsing destination: ", destination.destinationName);
 
      foreach (hotel; parallel(hotels)) {
          sqls.put(hotel.generateSql(destIdInDb).data.join);
      }
 
      sqls.data.writeln;
 
 
      delete fileContents;
      delete xmlFile;
    }
 
 }
 
 this is the full trace i got [1]:
 
 #0  0x0000000000bef5ef in gc.impl.conservative.gc.Gcx.smallAlloc(ubyte, 
 ref ulong, uint) ()
 [Current thread is 1 (LWP 100171)]
 (gdb) bt full
 #0  0x0000000000bef5ef in gc.impl.conservative.gc.Gcx.smallAlloc(ubyte, 
 ref ulong, uint) ()
 No symbol table info available.
 #1  0x0000000000bf3925 in 
 gc.impl.conservative.gc.ConservativeGC.runLocked!(gc.impl.conservative.gc.ConservativeGC
mallocNoSync(ulong, 
 uint, ref ulong, const(TypeInfo)), gc.impl.conservative.gc.mallocTime, 
 gc.impl.conservative.gc.numMallocs, ulong, uint, ulong, 
 const(TypeInfo)).runLocked(ref ulong, ref uint, ref ulong, ref 
 const(TypeInfo)) ()
 No symbol table info available.
 #2  0x0000000000bed103 in 
 gc.impl.conservative.gc.ConservativeGC.qalloc(ulong, uint, 
 const(TypeInfo)) ()
 No symbol table info available.
 #3  0x0000000000b9c6e3 in gc_qalloc ()
 No symbol table info available.
 #4  0x0000000000b96140 in core.memory.GC.qalloc(ulong, uint, 
 const(TypeInfo)) ()
 No symbol table info available.
 #5  0x000000000093fdce in 
 std.array.Appender!(immutable(char)[]).Appender.ensureAddable(ulong) 
 (this=..., nelems=761)
      at 
 /home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:2929
          len = 394
          reqlen = 1155
          newlen = 1155
          u = 0
          overflow = false
          nbytes = 1155
          bi = {base = 0x2b1, size = 140737488337376, attr = 4294949280}
 #6  0x0000000000942c0f in 
 std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).put(immutable(char)[
).bigDataFun(ulong) 
 (
      this=0x7fffffffb9f0, extra=761)
      at 
 /home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3023
 No locals.
 #7  0x0000000000942b55 in 
 std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).pu
(immutable(char)[]) 
 (this=...,
 ---Type <return> to continue, or q <return> to quit---
       at 
 /home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3026
          bigData = "'"
          len = 760
          newlen = 34799909888
not big deal probably, but isn't ~32GB enormous value here? I would check why bigDataFun return this.
Sep 26
parent Mengu <mengukagan gmail.com> writes:
On Tuesday, 26 September 2017 at 17:06:28 UTC, drug wrote:
 26.09.2017 00:34, Mengu пишет:
  [...]
not big deal probably, but isn't ~32GB enormous value here? I would check why bigDataFun return this.
i could not find out why. d certainly needs to improve on freebsd. i don't think devs would take d seriously if it won't work on a major platform. for now, i switched to debian rather than rewriting the app. i had some invalid memory operation errors but i managed to get rid of them. i'll certainly port the app to another language and switch back to freebsd.
Sep 26