Можно ли восстановить mongodb после отключения питания на сервере?

Отключилось питание. Все упало. Монга не поднялась. Поврежден журнал. При --repair краш.

mongod --dbpath /var/lib/mongodb --repair
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] MongoDB starting : pid=6702 port=27017 dbpath=/var/lib/mongodb 64-bit host=nondb02.mm4.com
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] db version v3.2.12
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] git version: ef3e1bc78e997f0d9f22f45aeb1d8e3b6ac14a14
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] OpenSSL version: OpenSSL 1.0.2g  1 Mar 2016
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] allocator: tcmalloc
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] modules: none
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] build environment:
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten]     distmod: ubuntu1604
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten]     distarch: x86_64
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten]     target_arch: x86_64
2018-04-06T00:56:27.757+0000 I CONTROL  [initandlisten] options: { repair: true, storage: { dbPath: "/var/lib/mongodb" } }
2018-04-06T00:56:27.785+0000 I -        [initandlisten] Detected data files in /var/lib/mongodb created by the 'wiredTiger' storage engine, so setting the active storage engine to 'wiredTiger'.
2018-04-06T00:56:27.785+0000 W -        [initandlisten] Detected unclean shutdown - /var/lib/mongodb/mongod.lock is not empty.
2018-04-06T00:56:27.785+0000 W STORAGE  [initandlisten] Recovering data from the last clean checkpoint.
2018-04-06T00:56:27.785+0000 I STORAGE  [initandlisten] Detected WT journal files.  Running recovery from last checkpoint.
2018-04-06T00:56:27.785+0000 I STORAGE  [initandlisten] journal to nojournal transition config: create,cache_size=18G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
2018-04-06T00:56:27.827+0000 E STORAGE  [initandlisten] WiredTiger (0) [1522976187:827578][6702:0x7fd30dac7c80], file:WiredTiger.wt, connection: read checksum error for 4096B block at offset 118784: block header checksum of 1667457286 doesn't match expected checksum of 4182632751
2018-04-06T00:56:27.827+0000 E STORAGE  [initandlisten] WiredTiger (0) [1522976187:827651][6702:0x7fd30dac7c80], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
2018-04-06T00:56:27.827+0000 E STORAGE  [initandlisten] WiredTiger (-31804) [1522976187:827660][6702:0x7fd30dac7c80], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
2018-04-06T00:56:27.827+0000 I -        [initandlisten] Fatal Assertion 28558
2018-04-06T00:56:27.827+0000 I -        [initandlisten] 

***aborting after fassert() failure


2018-04-06T00:56:27.849+0000 F -        [initandlisten] Got signal: 6 (Aborted).

 0x1533852 0x15327f9 0x1533062 0x7fd30c6cf390 0x7fd30c32a428 0x7fd30c32c02a 0x14b57f3 0x125b4d2 0x96f272 0x96f343 0x96f509 0x1b7681f 0x1b76d7b 0x1b7371d 0x1b77eb7 0x1b94e6c 0x1bca33b 0x1c52eb2 0x1c53585 0x1c53a7c 0x1bd9ac1 0x1c48d68 0x1c148df 0x1c149ac 0x1bc6816 0x123f35e 0x1238735 0x1133f38 0x9bba60 0x970392 0x7fd30c315830 0x9b6089
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"1133852","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"11327F9"},{"b":"400000","o":"1133062"},{"b":"7FD30C6BE000","o":"11390"},{"b":"7FD30C2F5000","o":"35428","s":"gsignal"},{"b":"7FD30C2F5000","o":"3702A","s":"abort"},{"b":"400000","o":"10B57F3","s":"_ZN5mongo13fassertFailedEi"},{"b":"400000","o":"E5B4D2"},{"b":"400000","o":"56F272"},{"b":"400000","o":"56F343","s":"__wt_err"},{"b":"400000","o":"56F509","s":"__wt_panic"},{"b":"400000","o":"177681F"},{"b":"400000","o":"1776D7B"},{"b":"400000","o":"177371D"},{"b":"400000","o":"1777EB7"},{"b":"400000","o":"1794E6C"},{"b":"400000","o":"17CA33B"},{"b":"400000","o":"1852EB2"},{"b":"400000","o":"1853585"},{"b":"400000","o":"1853A7C"},{"b":"400000","o":"17D9AC1"},{"b":"400000","o":"1848D68"},{"b":"400000","o":"18148DF"},{"b":"400000","o":"18149AC"},{"b":"400000","o":"17C6816","s":"wiredtiger_open"},{"b":"400000","o":"E3F35E","s":"_ZN5mongo18WiredTigerKVEngineC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_S8_mbbb"},{"b":"400000","o":"E38735"},{"b":"400000","o":"D33F38","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"400000","o":"5BBA60","s":"_ZN5mongo13initAndListenEi"},{"b":"400000","o":"570392","s":"main"},{"b":"7FD30C2F5000","o":"20830","s":"__libc_start_main"},{"b":"400000","o":"5B6089","s":"_start"}],"processInfo":{
 "mongodbVersion" : "3.2.12", "gitVersion" : "ef3e1bc78e997f0d9f22f45aeb1d8e3b6ac14a14", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.0-66-generic", "version" : "#87-Ubuntu SMP Fri Mar 3 15:29:05 UTC 2017", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "7877295E4A95F29C74796963664BD1115CFE251E" }, { "b" : "7FFE7C49E000", "elfType" : 3, "buildId" : "5CCDAB3B3062DC906D26EBB5A688D41BFE5EFD75" }, { "b" : "7FD30D64A000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "7F514146540382F59AD705BA8C913A75204C6858" }, { "b" : "7FD30D206000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "E6D4D2E4A048992CD5501E5985094E6CEC6C5D4F" }, { "b" : "7FD30CFFE000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "A656E8087D6226AE46A173477B8B8E0767D559D9" }, { "b" : "7FD30CDFA000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "AFDCB9B3774F5AFE6EB3EC9A76A8BA159A17D5B6" }, { "b" : "7FD30CAF1000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "4DBE0469DACBA00586428C9C7F4BBCEC8B6B1489" }, { "b" : "7FD30C8DB000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7FD30C6BE000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "4D7F52F335DC9665C2DCF308CE6514A6AE86DEDE" }, { "b" : "7FD30C2F5000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "60131540DADC6796CAB33388349E6E4E68692053" }, { "b" : "7FD30D8B3000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "EA8652B03788783033C3AF093DFB71509C1378F8" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1533852]
 mongod(+0x11327F9) [0x15327f9]
 mongod(+0x1133062) [0x1533062]
 libpthread.so.0(+0x11390) [0x7fd30c6cf390]
 libc.so.6(gsignal+0x38) [0x7fd30c32a428]
 libc.so.6(abort+0x16A) [0x7fd30c32c02a]
 mongod(_ZN5mongo13fassertFailedEi+0x93) [0x14b57f3]
 mongod(+0xE5B4D2) [0x125b4d2]
 mongod(+0x56F272) [0x96f272]
 mongod(__wt_err+0x8B) [0x96f343]
 mongod(__wt_panic+0x24) [0x96f509]
 mongod(+0x177681F) [0x1b7681f]
 mongod(+0x1776D7B) [0x1b76d7b]
 mongod(+0x177371D) [0x1b7371d]
 mongod(+0x1777EB7) [0x1b77eb7]
 mongod(+0x1794E6C) [0x1b94e6c]
 mongod(+0x17CA33B) [0x1bca33b]
 mongod(+0x1852EB2) [0x1c52eb2]
 mongod(+0x1853585) [0x1c53585]
 mongod(+0x1853A7C) [0x1c53a7c]
 mongod(+0x17D9AC1) [0x1bd9ac1]
 mongod(+0x1848D68) [0x1c48d68]
 mongod(+0x18148DF) [0x1c148df]
 mongod(+0x18149AC) [0x1c149ac]
 mongod(wiredtiger_open+0x15D6) [0x1bc6816]
 mongod(_ZN5mongo18WiredTigerKVEngineC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_S8_mbbb+0xBCE) [0x123f35e]
 mongod(+0xE38735) [0x1238735]
 mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x3E8) [0x1133f38]
 mongod(_ZN5mongo13initAndListenEi+0x480) [0x9bba60]
 mongod(main+0x712) [0x970392]
 libc.so.6(__libc_start_main+0xF0) [0x7fd30c315830]
 mongod(_start+0x29) [0x9b6089]
-----  END BACKTRACE  -----
Aborted (core dumped)


Пробую сделать дамп из коллекции, провал:
./wt -v -h ../mongodb/ -C "extensions=[./ext/compressors/snappy/.libs/libwiredtiger_snappy.so]" -R dump -f ../collection1.dump ../mongodb/collection-0-2820584975134648483.wt 
[1523011664:747357][25443:0x7f7399833700], file:WiredTiger.wt, connection: read checksum error for 4096B block at offset 118784: block header checksum of 1667457286 doesn't match expected checksum of 4182632751
[1523011664:747381][25443:0x7f7399833700], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
[1523011664:747385][25443:0x7f7399833700], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
lt-wt: WT_PANIC: WiredTiger library panic


Может быть есть еще способы или просто идеи как восстановить.

Бэкапов нет.
  • Вопрос задан
  • 1231 просмотр
Решения вопроса 1
@iArcher Автор вопроса
Результат исследований показал, что помочь в такой ситуации может обращение в сапорт монго ( https://jira.mongodb.org ). Создать тикет и описать свою ситуацию. С большой долей вероятности ребята пофиксят фалы WiredTiger.wt и WiredTiger.turtle, что может помочь.
В моем конкретном случае, по мимо этих файлов еще оказались повреждены файлы с коллекциями, что означает, что база воскрешению не подлежит.
Делайте бэкапы господа.
Ответ написан
Комментировать
Пригласить эксперта
Ваш ответ на вопрос

Войдите, чтобы написать ответ

Войти через центр авторизации
Похожие вопросы