| 1 | <?xml version="1.0" encoding="utf-8" ?>
|
|---|
| 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|---|
| 3 | <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
|---|
| 4 | <head>
|
|---|
| 5 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|---|
| 6 | <meta name="generator" content="Docutils 0.12: http://docutils.sourceforge.net/" />
|
|---|
| 7 | <title></title>
|
|---|
| 8 | <style type="text/css">
|
|---|
| 9 |
|
|---|
| 10 | /*
|
|---|
| 11 | :Author: David Goodger (goodger@python.org)
|
|---|
| 12 | :Id: $Id: TestBoxImaging.html 69111 2017-10-17 14:26:02Z vboxsync $
|
|---|
| 13 | :Copyright: This stylesheet has been placed in the public domain.
|
|---|
| 14 |
|
|---|
| 15 | Default cascading style sheet for the HTML output of Docutils.
|
|---|
| 16 |
|
|---|
| 17 | See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to
|
|---|
| 18 | customize this style sheet.
|
|---|
| 19 | */
|
|---|
| 20 |
|
|---|
| 21 | /* used to remove borders from tables and images */
|
|---|
| 22 | .borderless, table.borderless td, table.borderless th {
|
|---|
| 23 | border: 0 }
|
|---|
| 24 |
|
|---|
| 25 | table.borderless td, table.borderless th {
|
|---|
| 26 | /* Override padding for "table.docutils td" with "! important".
|
|---|
| 27 | The right padding separates the table cells. */
|
|---|
| 28 | padding: 0 0.5em 0 0 ! important }
|
|---|
| 29 |
|
|---|
| 30 | .first {
|
|---|
| 31 | /* Override more specific margin styles with "! important". */
|
|---|
| 32 | margin-top: 0 ! important }
|
|---|
| 33 |
|
|---|
| 34 | .last, .with-subtitle {
|
|---|
| 35 | margin-bottom: 0 ! important }
|
|---|
| 36 |
|
|---|
| 37 | .hidden {
|
|---|
| 38 | display: none }
|
|---|
| 39 |
|
|---|
| 40 | a.toc-backref {
|
|---|
| 41 | text-decoration: none ;
|
|---|
| 42 | color: black }
|
|---|
| 43 |
|
|---|
| 44 | blockquote.epigraph {
|
|---|
| 45 | margin: 2em 5em ; }
|
|---|
| 46 |
|
|---|
| 47 | dl.docutils dd {
|
|---|
| 48 | margin-bottom: 0.5em }
|
|---|
| 49 |
|
|---|
| 50 | object[type="image/svg+xml"], object[type="application/x-shockwave-flash"] {
|
|---|
| 51 | overflow: hidden;
|
|---|
| 52 | }
|
|---|
| 53 |
|
|---|
| 54 | /* Uncomment (and remove this text!) to get bold-faced definition list terms
|
|---|
| 55 | dl.docutils dt {
|
|---|
| 56 | font-weight: bold }
|
|---|
| 57 | */
|
|---|
| 58 |
|
|---|
| 59 | div.abstract {
|
|---|
| 60 | margin: 2em 5em }
|
|---|
| 61 |
|
|---|
| 62 | div.abstract p.topic-title {
|
|---|
| 63 | font-weight: bold ;
|
|---|
| 64 | text-align: center }
|
|---|
| 65 |
|
|---|
| 66 | div.admonition, div.attention, div.caution, div.danger, div.error,
|
|---|
| 67 | div.hint, div.important, div.note, div.tip, div.warning {
|
|---|
| 68 | margin: 2em ;
|
|---|
| 69 | border: medium outset ;
|
|---|
| 70 | padding: 1em }
|
|---|
| 71 |
|
|---|
| 72 | div.admonition p.admonition-title, div.hint p.admonition-title,
|
|---|
| 73 | div.important p.admonition-title, div.note p.admonition-title,
|
|---|
| 74 | div.tip p.admonition-title {
|
|---|
| 75 | font-weight: bold ;
|
|---|
| 76 | font-family: sans-serif }
|
|---|
| 77 |
|
|---|
| 78 | div.attention p.admonition-title, div.caution p.admonition-title,
|
|---|
| 79 | div.danger p.admonition-title, div.error p.admonition-title,
|
|---|
| 80 | div.warning p.admonition-title, .code .error {
|
|---|
| 81 | color: red ;
|
|---|
| 82 | font-weight: bold ;
|
|---|
| 83 | font-family: sans-serif }
|
|---|
| 84 |
|
|---|
| 85 | /* Uncomment (and remove this text!) to get reduced vertical space in
|
|---|
| 86 | compound paragraphs.
|
|---|
| 87 | div.compound .compound-first, div.compound .compound-middle {
|
|---|
| 88 | margin-bottom: 0.5em }
|
|---|
| 89 |
|
|---|
| 90 | div.compound .compound-last, div.compound .compound-middle {
|
|---|
| 91 | margin-top: 0.5em }
|
|---|
| 92 | */
|
|---|
| 93 |
|
|---|
| 94 | div.dedication {
|
|---|
| 95 | margin: 2em 5em ;
|
|---|
| 96 | text-align: center ;
|
|---|
| 97 | font-style: italic }
|
|---|
| 98 |
|
|---|
| 99 | div.dedication p.topic-title {
|
|---|
| 100 | font-weight: bold ;
|
|---|
| 101 | font-style: normal }
|
|---|
| 102 |
|
|---|
| 103 | div.figure {
|
|---|
| 104 | margin-left: 2em ;
|
|---|
| 105 | margin-right: 2em }
|
|---|
| 106 |
|
|---|
| 107 | div.footer, div.header {
|
|---|
| 108 | clear: both;
|
|---|
| 109 | font-size: smaller }
|
|---|
| 110 |
|
|---|
| 111 | div.line-block {
|
|---|
| 112 | display: block ;
|
|---|
| 113 | margin-top: 1em ;
|
|---|
| 114 | margin-bottom: 1em }
|
|---|
| 115 |
|
|---|
| 116 | div.line-block div.line-block {
|
|---|
| 117 | margin-top: 0 ;
|
|---|
| 118 | margin-bottom: 0 ;
|
|---|
| 119 | margin-left: 1.5em }
|
|---|
| 120 |
|
|---|
| 121 | div.sidebar {
|
|---|
| 122 | margin: 0 0 0.5em 1em ;
|
|---|
| 123 | border: medium outset ;
|
|---|
| 124 | padding: 1em ;
|
|---|
| 125 | background-color: #ffffee ;
|
|---|
| 126 | width: 40% ;
|
|---|
| 127 | float: right ;
|
|---|
| 128 | clear: right }
|
|---|
| 129 |
|
|---|
| 130 | div.sidebar p.rubric {
|
|---|
| 131 | font-family: sans-serif ;
|
|---|
| 132 | font-size: medium }
|
|---|
| 133 |
|
|---|
| 134 | div.system-messages {
|
|---|
| 135 | margin: 5em }
|
|---|
| 136 |
|
|---|
| 137 | div.system-messages h1 {
|
|---|
| 138 | color: red }
|
|---|
| 139 |
|
|---|
| 140 | div.system-message {
|
|---|
| 141 | border: medium outset ;
|
|---|
| 142 | padding: 1em }
|
|---|
| 143 |
|
|---|
| 144 | div.system-message p.system-message-title {
|
|---|
| 145 | color: red ;
|
|---|
| 146 | font-weight: bold }
|
|---|
| 147 |
|
|---|
| 148 | div.topic {
|
|---|
| 149 | margin: 2em }
|
|---|
| 150 |
|
|---|
| 151 | h1.section-subtitle, h2.section-subtitle, h3.section-subtitle,
|
|---|
| 152 | h4.section-subtitle, h5.section-subtitle, h6.section-subtitle {
|
|---|
| 153 | margin-top: 0.4em }
|
|---|
| 154 |
|
|---|
| 155 | h1.title {
|
|---|
| 156 | text-align: center }
|
|---|
| 157 |
|
|---|
| 158 | h2.subtitle {
|
|---|
| 159 | text-align: center }
|
|---|
| 160 |
|
|---|
| 161 | hr.docutils {
|
|---|
| 162 | width: 75% }
|
|---|
| 163 |
|
|---|
| 164 | img.align-left, .figure.align-left, object.align-left {
|
|---|
| 165 | clear: left ;
|
|---|
| 166 | float: left ;
|
|---|
| 167 | margin-right: 1em }
|
|---|
| 168 |
|
|---|
| 169 | img.align-right, .figure.align-right, object.align-right {
|
|---|
| 170 | clear: right ;
|
|---|
| 171 | float: right ;
|
|---|
| 172 | margin-left: 1em }
|
|---|
| 173 |
|
|---|
| 174 | img.align-center, .figure.align-center, object.align-center {
|
|---|
| 175 | display: block;
|
|---|
| 176 | margin-left: auto;
|
|---|
| 177 | margin-right: auto;
|
|---|
| 178 | }
|
|---|
| 179 |
|
|---|
| 180 | .align-left {
|
|---|
| 181 | text-align: left }
|
|---|
| 182 |
|
|---|
| 183 | .align-center {
|
|---|
| 184 | clear: both ;
|
|---|
| 185 | text-align: center }
|
|---|
| 186 |
|
|---|
| 187 | .align-right {
|
|---|
| 188 | text-align: right }
|
|---|
| 189 |
|
|---|
| 190 | /* reset inner alignment in figures */
|
|---|
| 191 | div.align-right {
|
|---|
| 192 | text-align: inherit }
|
|---|
| 193 |
|
|---|
| 194 | /* div.align-center * { */
|
|---|
| 195 | /* text-align: left } */
|
|---|
| 196 |
|
|---|
| 197 | ol.simple, ul.simple {
|
|---|
| 198 | margin-bottom: 1em }
|
|---|
| 199 |
|
|---|
| 200 | ol.arabic {
|
|---|
| 201 | list-style: decimal }
|
|---|
| 202 |
|
|---|
| 203 | ol.loweralpha {
|
|---|
| 204 | list-style: lower-alpha }
|
|---|
| 205 |
|
|---|
| 206 | ol.upperalpha {
|
|---|
| 207 | list-style: upper-alpha }
|
|---|
| 208 |
|
|---|
| 209 | ol.lowerroman {
|
|---|
| 210 | list-style: lower-roman }
|
|---|
| 211 |
|
|---|
| 212 | ol.upperroman {
|
|---|
| 213 | list-style: upper-roman }
|
|---|
| 214 |
|
|---|
| 215 | p.attribution {
|
|---|
| 216 | text-align: right ;
|
|---|
| 217 | margin-left: 50% }
|
|---|
| 218 |
|
|---|
| 219 | p.caption {
|
|---|
| 220 | font-style: italic }
|
|---|
| 221 |
|
|---|
| 222 | p.credits {
|
|---|
| 223 | font-style: italic ;
|
|---|
| 224 | font-size: smaller }
|
|---|
| 225 |
|
|---|
| 226 | p.label {
|
|---|
| 227 | white-space: nowrap }
|
|---|
| 228 |
|
|---|
| 229 | p.rubric {
|
|---|
| 230 | font-weight: bold ;
|
|---|
| 231 | font-size: larger ;
|
|---|
| 232 | color: maroon ;
|
|---|
| 233 | text-align: center }
|
|---|
| 234 |
|
|---|
| 235 | p.sidebar-title {
|
|---|
| 236 | font-family: sans-serif ;
|
|---|
| 237 | font-weight: bold ;
|
|---|
| 238 | font-size: larger }
|
|---|
| 239 |
|
|---|
| 240 | p.sidebar-subtitle {
|
|---|
| 241 | font-family: sans-serif ;
|
|---|
| 242 | font-weight: bold }
|
|---|
| 243 |
|
|---|
| 244 | p.topic-title {
|
|---|
| 245 | font-weight: bold }
|
|---|
| 246 |
|
|---|
| 247 | pre.address {
|
|---|
| 248 | margin-bottom: 0 ;
|
|---|
| 249 | margin-top: 0 ;
|
|---|
| 250 | font: inherit }
|
|---|
| 251 |
|
|---|
| 252 | pre.literal-block, pre.doctest-block, pre.math, pre.code {
|
|---|
| 253 | margin-left: 2em ;
|
|---|
| 254 | margin-right: 2em }
|
|---|
| 255 |
|
|---|
| 256 | pre.code .ln { color: grey; } /* line numbers */
|
|---|
| 257 | pre.code, code { background-color: #eeeeee }
|
|---|
| 258 | pre.code .comment, code .comment { color: #5C6576 }
|
|---|
| 259 | pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }
|
|---|
| 260 | pre.code .literal.string, code .literal.string { color: #0C5404 }
|
|---|
| 261 | pre.code .name.builtin, code .name.builtin { color: #352B84 }
|
|---|
| 262 | pre.code .deleted, code .deleted { background-color: #DEB0A1}
|
|---|
| 263 | pre.code .inserted, code .inserted { background-color: #A3D289}
|
|---|
| 264 |
|
|---|
| 265 | span.classifier {
|
|---|
| 266 | font-family: sans-serif ;
|
|---|
| 267 | font-style: oblique }
|
|---|
| 268 |
|
|---|
| 269 | span.classifier-delimiter {
|
|---|
| 270 | font-family: sans-serif ;
|
|---|
| 271 | font-weight: bold }
|
|---|
| 272 |
|
|---|
| 273 | span.interpreted {
|
|---|
| 274 | font-family: sans-serif }
|
|---|
| 275 |
|
|---|
| 276 | span.option {
|
|---|
| 277 | white-space: nowrap }
|
|---|
| 278 |
|
|---|
| 279 | span.pre {
|
|---|
| 280 | white-space: pre }
|
|---|
| 281 |
|
|---|
| 282 | span.problematic {
|
|---|
| 283 | color: red }
|
|---|
| 284 |
|
|---|
| 285 | span.section-subtitle {
|
|---|
| 286 | /* font-size relative to parent (h1..h6 element) */
|
|---|
| 287 | font-size: 80% }
|
|---|
| 288 |
|
|---|
| 289 | table.citation {
|
|---|
| 290 | border-left: solid 1px gray;
|
|---|
| 291 | margin-left: 1px }
|
|---|
| 292 |
|
|---|
| 293 | table.docinfo {
|
|---|
| 294 | margin: 2em 4em }
|
|---|
| 295 |
|
|---|
| 296 | table.docutils {
|
|---|
| 297 | margin-top: 0.5em ;
|
|---|
| 298 | margin-bottom: 0.5em }
|
|---|
| 299 |
|
|---|
| 300 | table.footnote {
|
|---|
| 301 | border-left: solid 1px black;
|
|---|
| 302 | margin-left: 1px }
|
|---|
| 303 |
|
|---|
| 304 | table.docutils td, table.docutils th,
|
|---|
| 305 | table.docinfo td, table.docinfo th {
|
|---|
| 306 | padding-left: 0.5em ;
|
|---|
| 307 | padding-right: 0.5em ;
|
|---|
| 308 | vertical-align: top }
|
|---|
| 309 |
|
|---|
| 310 | table.docutils th.field-name, table.docinfo th.docinfo-name {
|
|---|
| 311 | font-weight: bold ;
|
|---|
| 312 | text-align: left ;
|
|---|
| 313 | white-space: nowrap ;
|
|---|
| 314 | padding-left: 0 }
|
|---|
| 315 |
|
|---|
| 316 | /* "booktabs" style (no vertical lines) */
|
|---|
| 317 | table.docutils.booktabs {
|
|---|
| 318 | border: 0px;
|
|---|
| 319 | border-top: 2px solid;
|
|---|
| 320 | border-bottom: 2px solid;
|
|---|
| 321 | border-collapse: collapse;
|
|---|
| 322 | }
|
|---|
| 323 | table.docutils.booktabs * {
|
|---|
| 324 | border: 0px;
|
|---|
| 325 | }
|
|---|
| 326 | table.docutils.booktabs th {
|
|---|
| 327 | border-bottom: thin solid;
|
|---|
| 328 | text-align: left;
|
|---|
| 329 | }
|
|---|
| 330 |
|
|---|
| 331 | h1 tt.docutils, h2 tt.docutils, h3 tt.docutils,
|
|---|
| 332 | h4 tt.docutils, h5 tt.docutils, h6 tt.docutils {
|
|---|
| 333 | font-size: 100% }
|
|---|
| 334 |
|
|---|
| 335 | ul.auto-toc {
|
|---|
| 336 | list-style-type: none }
|
|---|
| 337 |
|
|---|
| 338 | </style>
|
|---|
| 339 | </head>
|
|---|
| 340 | <body>
|
|---|
| 341 | <div class="document">
|
|---|
| 342 |
|
|---|
| 343 |
|
|---|
| 344 | <div class="section" id="testbox-imaging-backup-restore">
|
|---|
| 345 | <h1>Testbox Imaging (Backup / Restore)</h1>
|
|---|
| 346 | <div class="section" id="introduction">
|
|---|
| 347 | <h2>Introduction</h2>
|
|---|
| 348 | <p>This document is explores deloying a very simple drive imaging solution to help
|
|---|
| 349 | avoid needing to manually reinstall testboxes when a disk goes bust or the OS
|
|---|
| 350 | install seems to be corrupted.</p>
|
|---|
| 351 | </div>
|
|---|
| 352 | </div>
|
|---|
| 353 | <div class="section" id="definitions-glossary">
|
|---|
| 354 | <h1>Definitions / Glossary</h1>
|
|---|
| 355 | <p>See AutomaticTestingRevamp.txt.</p>
|
|---|
| 356 | </div>
|
|---|
| 357 | <div class="section" id="objectives">
|
|---|
| 358 | <h1>Objectives</h1>
|
|---|
| 359 | <blockquote>
|
|---|
| 360 | <ul class="simple">
|
|---|
| 361 | <li>Off site, no admin interaction (no need for ILOM or similar).</li>
|
|---|
| 362 | <li>OS independent.</li>
|
|---|
| 363 | <li>Space and bandwidth efficient.</li>
|
|---|
| 364 | <li>As automatic as possible.</li>
|
|---|
| 365 | <li>Logging.</li>
|
|---|
| 366 | </ul>
|
|---|
| 367 | </blockquote>
|
|---|
| 368 | </div>
|
|---|
| 369 | <div class="section" id="overview-of-the-solution">
|
|---|
| 370 | <h1>Overview of the Solution</h1>
|
|---|
| 371 | <p>Here is a brief summary:</p>
|
|---|
| 372 | <blockquote>
|
|---|
| 373 | <ul class="simple">
|
|---|
| 374 | <li>Always boot testboxes via PXE using PXELINUX.</li>
|
|---|
| 375 | <li>Default configuration is local boot (hard disk / SSD)</li>
|
|---|
| 376 | <li>Restore/backup action triggered by machine specific PXE config.</li>
|
|---|
| 377 | <li>Boots special debian maintenance install off NFS.</li>
|
|---|
| 378 | <li>A maintenance service (systemd style) does the work.</li>
|
|---|
| 379 | <li>The service reads action from TFTP location and performs it.</li>
|
|---|
| 380 | <li>When done the service removes the TFTP machine specific config
|
|---|
| 381 | and reboots the system.</li>
|
|---|
| 382 | </ul>
|
|---|
| 383 | </blockquote>
|
|---|
| 384 | <dl class="docutils">
|
|---|
| 385 | <dt>Maintenance actions are:</dt>
|
|---|
| 386 | <dd><ul class="first last simple">
|
|---|
| 387 | <li>backup</li>
|
|---|
| 388 | <li>backup-again</li>
|
|---|
| 389 | <li>restore</li>
|
|---|
| 390 | <li>refresh-info</li>
|
|---|
| 391 | <li>rescue</li>
|
|---|
| 392 | </ul>
|
|---|
| 393 | </dd>
|
|---|
| 394 | </dl>
|
|---|
| 395 | <p>Possible modifier that indicates a subset of disk on testboxes with other OSes
|
|---|
| 396 | installed. Support for partition level backup/restore is not explored here.</p>
|
|---|
| 397 | <div class="section" id="how-to-use">
|
|---|
| 398 | <h2>How to use</h2>
|
|---|
| 399 | <p>To perform one of the above maintenance actions on a testbox, run the
|
|---|
| 400 | <tt class="docutils literal"><span class="pre">testbox-pxe-conf.sh</span></tt> script:</p>
|
|---|
| 401 | <pre class="literal-block">
|
|---|
| 402 | /mnt/testbox-tftp/pxeclient.cfg/testbox-pxe-conf.sh 10.165.98.220 rescue
|
|---|
| 403 | </pre>
|
|---|
| 404 | <p>Then trigger a reboot. The box will then boot the NFS rooted debian image and
|
|---|
| 405 | execute the maintenance action. On success, it will remove the testbox hex-IP
|
|---|
| 406 | config file and reboot again.</p>
|
|---|
| 407 | </div>
|
|---|
| 408 | </div>
|
|---|
| 409 | <div class="section" id="storage-server">
|
|---|
| 410 | <h1>Storage Server</h1>
|
|---|
| 411 | <p>The storage server will have three areas used here. Using NFS for all three
|
|---|
| 412 | avoids extra work getting CIFS sharing right too (NFS is already a pain).</p>
|
|---|
| 413 | <blockquote>
|
|---|
| 414 | <ol class="arabic simple">
|
|---|
| 415 | <li>/export/testbox-tftp - TFTP config area. Read-write.</li>
|
|---|
| 416 | <li>/export/testbox-backup - Images and logs. Read-write.</li>
|
|---|
| 417 | <li>/export/testbox-nfsroot - Custom debian. Read-only, no root squash.</li>
|
|---|
| 418 | </ol>
|
|---|
| 419 | </blockquote>
|
|---|
| 420 | </div>
|
|---|
| 421 | <div class="section" id="tftp-export-testbox-tftp">
|
|---|
| 422 | <h1>TFTP (/export/testbox-tftp)</h1>
|
|---|
| 423 | <p>The testbox-tftp share needs to be writable, root squashing is okay.</p>
|
|---|
| 424 | <p>We need files from both PXELINUX and SYSLINUX to make this work now. On a
|
|---|
| 425 | debian system, the <tt class="docutils literal">pxelinux</tt> and <tt class="docutils literal">syslinux</tt> packages needs to be
|
|---|
| 426 | installed. We actually do this further down when setting up the nfsroot, so
|
|---|
| 427 | it's possible to get them from there by postponing this step a little. On
|
|---|
| 428 | debian 8.6.0 the PXELINUX files are found in <tt class="docutils literal">/usr/lib/PXELINUX</tt> and the
|
|---|
| 429 | SYSLINUX ones in <tt class="docutils literal">/usr/lib/syslinux</tt>.</p>
|
|---|
| 430 | <p>The initial PXE image as well as associated modules comes in three variants,
|
|---|
| 431 | BIOS, 32-bit EFI and 64-bit EFI. We'll only need the BIOS one for now.
|
|---|
| 432 | Perform the following copy operations:</p>
|
|---|
| 433 | <pre class="literal-block">
|
|---|
| 434 | cp /usr/lib/PXELINUX/pxelinux.0 /mnt/testbox-tftp/
|
|---|
| 435 | cp /usr/lib/syslinux/modules/*/ldlinux.* /mnt/testbox-tftp/
|
|---|
| 436 | cp -R /usr/lib/syslinux/modules/bios /mnt/testbox-tftp/
|
|---|
| 437 | cp -R /usr/lib/syslinux/modules/efi32 /mnt/testbox-tftp/
|
|---|
| 438 | cp -R /usr/lib/syslinux/modules/efi64 /mnt/testbox-tftp/
|
|---|
| 439 | </pre>
|
|---|
| 440 | <p>For simplicitly, all the testboxes boot using good old fashioned BIOS, no EFI.
|
|---|
| 441 | However, it doesn't really hurt to be prepared.</p>
|
|---|
| 442 | <p>The PXELINUX related files goes in the root of the testbox-tftp share. (As
|
|---|
| 443 | mentioned further down, these can be installed on a debian system by running
|
|---|
| 444 | <tt class="docutils literal"><span class="pre">apt-get</span> install pxelinux syslinux</tt>.) We need the <tt class="docutils literal">*pxelinux.0</tt> files
|
|---|
| 445 | typically found in <tt class="docutils literal">/usr/lib/PXELINUX/</tt> on debian systems (recent ones
|
|---|
| 446 | anyway). It is possible we may need one ore more fo the modules <a class="footnote-reference" href="#id6" id="id1">[1]</a> that
|
|---|
| 447 | ships with PXELINUX/SYSLINUX, so do copy <tt class="docutils literal">/usr/lib/syslinux/modules</tt> to
|
|---|
| 448 | <tt class="docutils literal"><span class="pre">testbox-tftp/modules</span></tt> as well.</p>
|
|---|
| 449 | <p>The directory layout related to the configuration files is dictated by the
|
|---|
| 450 | PXELINUX configuration file searching algorithm <a class="footnote-reference" href="#id7" id="id2">[2]</a>. Create a subdirectory
|
|---|
| 451 | <tt class="docutils literal">pxelinux.cfg/</tt> under <tt class="docutils literal"><span class="pre">testbox-tftp</span></tt> and create the world readable file
|
|---|
| 452 | <tt class="docutils literal">default</tt> with the following content:</p>
|
|---|
| 453 | <pre class="literal-block">
|
|---|
| 454 | PATH bios
|
|---|
| 455 | DEFAULT local-boot
|
|---|
| 456 | LABEL local-boot
|
|---|
| 457 | LOCALBOOT
|
|---|
| 458 | </pre>
|
|---|
| 459 | <p>This will make the default behavior to boot the local disk system.</p>
|
|---|
| 460 | <p>Copy the <tt class="docutils literal"><span class="pre">testbox-pxe-conf.sh</span></tt> script file found in the same directory as
|
|---|
| 461 | this document to <tt class="docutils literal"><span class="pre">/mnt/testbox-tftp/pxelinux.cfg/</span></tt>. Edit the copy to correct
|
|---|
| 462 | the IP addresses near the top, as well as any linux, TFTP and PXE details near
|
|---|
| 463 | the bottom of the file. This script will generate the PXE configuration file
|
|---|
| 464 | when performing maintenance on a testbox.</p>
|
|---|
| 465 | </div>
|
|---|
| 466 | <div class="section" id="images-and-logs-export-testbox-backup">
|
|---|
| 467 | <h1>Images and logs (/export/testbox-backup)</h1>
|
|---|
| 468 | <p>The testbox-backup share needs to be writable, root squashing is okay.</p>
|
|---|
| 469 | <p>In the root there must be a file <tt class="docutils literal"><span class="pre">testbox-backup</span></tt> so we can easily tell
|
|---|
| 470 | whether we've actually mounted the share or are just staring at an empty mount
|
|---|
| 471 | point directory.</p>
|
|---|
| 472 | <p>The <tt class="docutils literal"><span class="pre">testbox-maintenance.sh</span></tt> script maintains a global log in the root
|
|---|
| 473 | directory that's called <tt class="docutils literal">maintenance.log</tt>. Errors will be logged there as
|
|---|
| 474 | well as a ping and the action.</p>
|
|---|
| 475 | <p>We use a directory layout based on dotted decimal IP addresses here, so for a
|
|---|
| 476 | server with the IP 10.40.41.42 all its file will be under <tt class="docutils literal">10.40.41.42/</tt>:</p>
|
|---|
| 477 | <dl class="docutils">
|
|---|
| 478 | <dt><tt class="docutils literal"><hostname></tt></dt>
|
|---|
| 479 | <dd>The name of the testbox (empty file). Help finding a testbox by name.</dd>
|
|---|
| 480 | <dt><tt class="docutils literal"><span class="pre">testbox-info.txt</span></tt></dt>
|
|---|
| 481 | <dd>Information about the testbox. Starting off with the name, decimal IP,
|
|---|
| 482 | PXELINUX style hexadecimal IP, and more.</dd>
|
|---|
| 483 | <dt><tt class="docutils literal">maintenance.log</tt></dt>
|
|---|
| 484 | <dd>Maintenance log file recording what the maintenance service does.</dd>
|
|---|
| 485 | <dt><tt class="docutils literal"><span class="pre">disk-devices.lst</span></tt></dt>
|
|---|
| 486 | <dd>Optional list of disk devices to consider backuping up or restoring. This is
|
|---|
| 487 | intended for testboxes with additional disks that are used for other purposes
|
|---|
| 488 | and should touched.</dd>
|
|---|
| 489 | <dt><tt class="docutils literal">sda.raw.gz</tt></dt>
|
|---|
| 490 | <dd>The gzipped raw copy of the sda device of the testbox.</dd>
|
|---|
| 491 | <dt><tt class="docutils literal"><span class="pre">sd[bcdefgh].raw.gz</span></tt></dt>
|
|---|
| 492 | <dd>The gzipped raw copy sdb, sdc, sde, sdf, sdg, sdh, etc if any of them exists
|
|---|
| 493 | and are disks/SSDs.</dd>
|
|---|
| 494 | <dt>Note! If it turns out we can be certain to get a valid host name, we might just</dt>
|
|---|
| 495 | <dd>switch to use the hostname as the directory name instead of the IP.</dd>
|
|---|
| 496 | </dl>
|
|---|
| 497 | </div>
|
|---|
| 498 | <div class="section" id="debian-nfs-root-export-testbox-nfsroot">
|
|---|
| 499 | <h1>Debian NFS root (/export/testbox-nfsroot)</h1>
|
|---|
| 500 | <p>The testbox-nfsroot share should be read-only and must <strong>not</strong> have root
|
|---|
| 501 | squashing enabled. Also, make sure setting the set-uid-bit is allowed by the
|
|---|
| 502 | server, or <tt class="docutils literal">su` and ``sudo</tt> won't work</p>
|
|---|
| 503 | <p>There are several ways of creating a debian nfsroot, but since we've got a
|
|---|
| 504 | tool like VirtualBox around we've just installed it in a VM, prepared it,
|
|---|
| 505 | and copied it onto the NFS server share.</p>
|
|---|
| 506 | <p>As of writing debian 8.6.0 is current, so a minimal 64-bit install of it was
|
|---|
| 507 | done in a VM. After installation the following modifications was done:</p>
|
|---|
| 508 | <blockquote>
|
|---|
| 509 | <ul>
|
|---|
| 510 | <li><p class="first"><tt class="docutils literal"><span class="pre">apt-get</span> install pxelinux syslinux <span class="pre">initramfs-tools</span> zip gddrescue sudo joe</tt>
|
|---|
| 511 | and optionally <tt class="docutils literal"><span class="pre">apt-get</span> install smbclient <span class="pre">cifs-utils</span></tt>.</p>
|
|---|
| 512 | </li>
|
|---|
| 513 | <li><p class="first"><tt class="docutils literal">/etc/default/grub</tt> was modified to set <tt class="docutils literal">GRUB_CMDLINE_LINUX_DEFAULT</tt> to
|
|---|
| 514 | <tt class="docutils literal">""</tt> instead of <tt class="docutils literal">"quiet"</tt>. This allows us to see messages during boot
|
|---|
| 515 | and perhaps spot why something doesn't work on a testbox. Regenerate the
|
|---|
| 516 | grub configuration file by running <tt class="docutils literal"><span class="pre">update-grub</span></tt> afterwards.</p>
|
|---|
| 517 | </li>
|
|---|
| 518 | <li><p class="first"><tt class="docutils literal">/etc/sudoers</tt> was modified to allow the <tt class="docutils literal">vbox</tt> user use sudo without
|
|---|
| 519 | requring any password.</p>
|
|---|
| 520 | </li>
|
|---|
| 521 | <li><p class="first">Create the directory <tt class="docutils literal">/etc/systemd/system/getty@tty1.service.d</tt> and create
|
|---|
| 522 | the file <tt class="docutils literal">noclear.conf</tt> in it with the following content:</p>
|
|---|
| 523 | <pre class="literal-block">
|
|---|
| 524 | [Service]
|
|---|
| 525 | TTYVTDisallocate=no
|
|---|
| 526 | </pre>
|
|---|
| 527 | <p>This stops getty from clearing VT1 and let us see the tail of the boot up
|
|---|
| 528 | messages, which includes messages from the testbox-maintenance service.</p>
|
|---|
| 529 | </li>
|
|---|
| 530 | <li><p class="first">Mount the testbox-nfsroot under <tt class="docutils literal">/mnt/</tt> with write privileges. (The write
|
|---|
| 531 | privileges are temporary - don't forget to remove them later on.):</p>
|
|---|
| 532 | <pre class="literal-block">
|
|---|
| 533 | mount -t nfs myserver.com:/export/testbox-nfsroot
|
|---|
| 534 | </pre>
|
|---|
| 535 | <p>Note! Adding <tt class="docutils literal"><span class="pre">-o</span> nfsvers=3</tt> may help with some NTFv4 servers.</p>
|
|---|
| 536 | </li>
|
|---|
| 537 | <li><p class="first">Copy the debian root and dev file system onto nfsroot. If you have ssh
|
|---|
| 538 | access to the NFS server, the quickest way to do it is to use <tt class="docutils literal">tar</tt>:</p>
|
|---|
| 539 | <pre class="literal-block">
|
|---|
| 540 | tar -cz --one-file-system -f /mnt/testbox-maintenance-nfsroot.tar.gz . dev/
|
|---|
| 541 | </pre>
|
|---|
| 542 | <p>An alternative is <tt class="docutils literal">cp <span class="pre">-ax</span> . /mnt/. && cp <span class="pre">-ax</span> dev/. /mnt/dev/.</tt> but this
|
|---|
| 543 | is quite a bit slower, obviously.</p>
|
|---|
| 544 | </li>
|
|---|
| 545 | <li><p class="first">Edit <tt class="docutils literal">/etc/ssh/sshd_config</tt> setting <tt class="docutils literal">PermitRootLogin</tt> to <tt class="docutils literal">yes</tt> so we can ssh
|
|---|
| 546 | in as root later on.</p>
|
|---|
| 547 | </li>
|
|---|
| 548 | <li><p class="first">chroot into the nfsroot: <tt class="docutils literal">chroot /mnt/</tt></p>
|
|---|
| 549 | <blockquote>
|
|---|
| 550 | <ul>
|
|---|
| 551 | <li><p class="first"><tt class="docutils literal">mount <span class="pre">-o</span> proc proc /proc</tt></p>
|
|---|
| 552 | </li>
|
|---|
| 553 | <li><p class="first"><tt class="docutils literal">mount <span class="pre">-o</span> sysfs sysfs /sys</tt></p>
|
|---|
| 554 | </li>
|
|---|
| 555 | <li><p class="first"><tt class="docutils literal">mkdir <span class="pre">/mnt/testbox-tftp</span> <span class="pre">/mnt/testbox-backup</span></tt></p>
|
|---|
| 556 | </li>
|
|---|
| 557 | <li><p class="first">Recreate <tt class="docutils literal">/etc/fstab</tt> with:</p>
|
|---|
| 558 | <pre class="literal-block">
|
|---|
| 559 | proc /proc proc defaults 0 0
|
|---|
| 560 | /dev/nfs / nfs defaults 1 1
|
|---|
| 561 | 10.42.1.1:/export/testbox-tftp /mnt/testbox-tftp nfs tcp,nfsvers=3,noauto 2 2
|
|---|
| 562 | 10.42.1.1:/export/testbox-backup /mnt/testbox-backup nfs tcp,nfsvers=3,noauto 3 3
|
|---|
| 563 | </pre>
|
|---|
| 564 | <p>We use NFS version 3 as that works better for our NFS server and client,
|
|---|
| 565 | remove if not necessary. The <tt class="docutils literal">noauto</tt> option is to work around mount
|
|---|
| 566 | trouble during early bootup on some of our boxes.</p>
|
|---|
| 567 | </li>
|
|---|
| 568 | <li><p class="first">Do <tt class="docutils literal">mount <span class="pre">/mnt/testbox-tftp</span> && mount <span class="pre">/mnt/testbox-backup</span></tt> to mount the
|
|---|
| 569 | two shares. This may be a good time to execute the instructions in the
|
|---|
| 570 | sections above relating to these two shares.</p>
|
|---|
| 571 | </li>
|
|---|
| 572 | <li><p class="first">Edit <tt class="docutils literal"><span class="pre">/etc/initramfs-tools/initramfs.conf</span></tt> and change the <tt class="docutils literal">MODULES</tt>
|
|---|
| 573 | value from <tt class="docutils literal">most</tt> to <tt class="docutils literal">netboot</tt>.</p>
|
|---|
| 574 | </li>
|
|---|
| 575 | <li><p class="first">Append <tt class="docutils literal">aufs</tt> to <tt class="docutils literal"><span class="pre">/etc/initramfs-tools/modules</span></tt>. The advanced
|
|---|
| 576 | multi-layered unification filesystem (aufs) enables us to use a
|
|---|
| 577 | read-only NFS root. <a class="footnote-reference" href="#id8" id="id3">[3]</a> <a class="footnote-reference" href="#id9" id="id4">[4]</a> <a class="footnote-reference" href="#id10" id="id5">[5]</a></p>
|
|---|
| 578 | </li>
|
|---|
| 579 | <li><p class="first">Create <tt class="docutils literal"><span class="pre">/etc/initramfs-tools/scripts/init-bottom/00_aufs_init</span></tt> as
|
|---|
| 580 | an executable file with the following content:</p>
|
|---|
| 581 | <pre class="literal-block">
|
|---|
| 582 | #!/bin/sh
|
|---|
| 583 | # Don't run during update-initramfs:
|
|---|
| 584 | case "$1" in
|
|---|
| 585 | prereqs)
|
|---|
| 586 | exit 0;
|
|---|
| 587 | ;;
|
|---|
| 588 | esac
|
|---|
| 589 |
|
|---|
| 590 | modprobe aufs
|
|---|
| 591 | mkdir -p /ro /rw /aufs
|
|---|
| 592 | mount -t tmpfs tmpfs /rw -o noatime,mode=0755
|
|---|
| 593 | mount --move $rootmnt /ro
|
|---|
| 594 | mount -t aufs aufs /aufs -o noatime,dirs=/rw:/ro=ro
|
|---|
| 595 | mkdir -p /aufs/rw /aufs/ro
|
|---|
| 596 | mount --move /ro /aufs/ro
|
|---|
| 597 | mount --move /rw /aufs/rw
|
|---|
| 598 | mount --move /aufs /root
|
|---|
| 599 | exit 0
|
|---|
| 600 | </pre>
|
|---|
| 601 | </li>
|
|---|
| 602 | <li><p class="first">Update the init ramdisk: <tt class="docutils literal"><span class="pre">update-initramfs</span> <span class="pre">-u</span> <span class="pre">-k</span> all</tt></p>
|
|---|
| 603 | <dl class="docutils">
|
|---|
| 604 | <dt>Note! It may be necessary to do <tt class="docutils literal">mount <span class="pre">-t</span> tmpfs tmpfs /var/tmp</tt> to help</dt>
|
|---|
| 605 | <dd><p class="first last">this operation succeed.</p>
|
|---|
| 606 | </dd>
|
|---|
| 607 | </dl>
|
|---|
| 608 | </li>
|
|---|
| 609 | <li><p class="first">Copy <tt class="docutils literal">/boot</tt> to <tt class="docutils literal"><span class="pre">/mnt/testbox-tftp/maintenance-boot/</span></tt>.</p>
|
|---|
| 610 | </li>
|
|---|
| 611 | <li><p class="first">Copy the <tt class="docutils literal"><span class="pre">testbox-maintenance.sh</span></tt> file found in the same directory as this
|
|---|
| 612 | document to <tt class="docutils literal">/root/scripts/</tt> (need to create the dir) and make it
|
|---|
| 613 | executable.</p>
|
|---|
| 614 | </li>
|
|---|
| 615 | <li><p class="first">Create the systemd service file for the maintenance service as
|
|---|
| 616 | <tt class="docutils literal"><span class="pre">/etc/systemd/system/testbox-maintenance.service</span></tt> with the content:</p>
|
|---|
| 617 | <pre class="literal-block">
|
|---|
| 618 | [Unit]
|
|---|
| 619 | Description=Testbox Maintenance
|
|---|
| 620 | After=network.target
|
|---|
| 621 | Before=getty@tty1.service
|
|---|
| 622 |
|
|---|
| 623 | [Service]
|
|---|
| 624 | Type=oneshot
|
|---|
| 625 | RemainAfterExit=True
|
|---|
| 626 | ExecStart=/root/scripts/testbox-maintenance.sh
|
|---|
| 627 | ExecStartPre=/bin/echo -e \033%G
|
|---|
| 628 | ExecReload=/bin/kill -HUP $MAINPID
|
|---|
| 629 | WorkingDirectory=/tmp
|
|---|
| 630 | Environment=TERM=xterm
|
|---|
| 631 | StandardOutput=journal+console
|
|---|
| 632 |
|
|---|
| 633 | [Install]
|
|---|
| 634 | WantedBy=multi-user.target
|
|---|
| 635 | </pre>
|
|---|
| 636 | </li>
|
|---|
| 637 | <li><p class="first">Enable our service: <tt class="docutils literal">systemctl enable <span class="pre">/etc/systemd/system/testbox-maintenance.service</span></tt></p>
|
|---|
| 638 | </li>
|
|---|
| 639 | <li><p class="first">xxxx ... more ???</p>
|
|---|
| 640 | </li>
|
|---|
| 641 | <li><p class="first">Before leaving the chroot, do <tt class="docutils literal">mount /proc /sys <span class="pre">/mnt/testbox-*</span></tt>.</p>
|
|---|
| 642 | </li>
|
|---|
| 643 | </ul>
|
|---|
| 644 | </blockquote>
|
|---|
| 645 | </li>
|
|---|
| 646 | <li><p class="first">Testing the setup from a VM is kind of useful (if the nfs server can be
|
|---|
| 647 | convinced to accept root nfs mounts from non-privileged clinet ports):</p>
|
|---|
| 648 | <blockquote>
|
|---|
| 649 | <ul>
|
|---|
| 650 | <li><p class="first">Create a VM using the 64-bit debian profile. Let's call it "pxe-vm".</p>
|
|---|
| 651 | </li>
|
|---|
| 652 | <li><p class="first">Mount the TFTP share somewhere, like M: or /mnt/testbox-tftp.</p>
|
|---|
| 653 | </li>
|
|---|
| 654 | <li><p class="first">Reconfigure the NAT DHCP and TFTP bits:</p>
|
|---|
| 655 | <pre class="literal-block">
|
|---|
| 656 | VBoxManage setextradata pxe-vm VBoxInternal/PDM/DriverTransformations/pxe/AboveDriver NAT
|
|---|
| 657 | VBoxManage setextradata pxe-vm VBoxInternal/PDM/DriverTransformations/pxe/Action mergeconfig
|
|---|
| 658 | VBoxManage setextradata pxe-vm VBoxInternal/PDM/DriverTransformations/pxe/Config/TFTPPrefix M:/
|
|---|
| 659 | VBoxManage setextradata pxe-vm VBoxInternal/PDM/DriverTransformations/pxe/Config/BootFile pxelinux.0
|
|---|
| 660 | </pre>
|
|---|
| 661 | </li>
|
|---|
| 662 | <li><p class="first">Create the file <tt class="docutils literal"><span class="pre">testbox-tftp/pxelinux.cfg/0A00020F</span></tt> containing:</p>
|
|---|
| 663 | <pre class="literal-block">
|
|---|
| 664 | PATH bios
|
|---|
| 665 | DEFAULT maintenance
|
|---|
| 666 | LABEL maintenance
|
|---|
| 667 | MENU LABEL Maintenance (NFS)
|
|---|
| 668 | KERNEL maintenance-boot/vmlinuz-3.16.0-4-amd64
|
|---|
| 669 | APPEND initrd=maintenance-boot/initrd.img-3.16.0-4-amd64 ro ip=dhcp aufs=tmpfs \
|
|---|
| 670 | boot=nfs root=/dev/nfs nfsroot=10.42.1.1:/export/testbox-nfsroot
|
|---|
| 671 | LABEL local-boot
|
|---|
| 672 | LOCALBOOT
|
|---|
| 673 | </pre>
|
|---|
| 674 | </li>
|
|---|
| 675 | </ul>
|
|---|
| 676 | </blockquote>
|
|---|
| 677 | </li>
|
|---|
| 678 | </ul>
|
|---|
| 679 | </blockquote>
|
|---|
| 680 | </div>
|
|---|
| 681 | <div class="section" id="troubleshooting">
|
|---|
| 682 | <h1>Troubleshooting</h1>
|
|---|
| 683 | <dl class="docutils">
|
|---|
| 684 | <dt><tt class="docutils literal"><span class="pre">PXE-E11</span></tt> or something like <tt class="docutils literal">No ARP reply</tt></dt>
|
|---|
| 685 | <dd>You probably got the TFTP and DHCP on different machines. Try move the TFTP
|
|---|
| 686 | to the same machine as the DHCP, then the PXE stack won't have to do any
|
|---|
| 687 | additional ARP resolving. Google results suggest that a congested network
|
|---|
| 688 | could use the ARP reply to get lost. Our suspicion is that it might also be
|
|---|
| 689 | related to the PXE stack shipping with the NIC.</dd>
|
|---|
| 690 | </dl>
|
|---|
| 691 | <hr class="docutils" />
|
|---|
| 692 | <table class="docutils footnote" frame="void" id="id6" rules="none">
|
|---|
| 693 | <colgroup><col class="label" /><col /></colgroup>
|
|---|
| 694 | <tbody valign="top">
|
|---|
| 695 | <tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>See <a class="reference external" href="http://www.syslinux.org/wiki/index.php?title=Category:Modules">http://www.syslinux.org/wiki/index.php?title=Category:Modules</a></td></tr>
|
|---|
| 696 | </tbody>
|
|---|
| 697 | </table>
|
|---|
| 698 | <table class="docutils footnote" frame="void" id="id7" rules="none">
|
|---|
| 699 | <colgroup><col class="label" /><col /></colgroup>
|
|---|
| 700 | <tbody valign="top">
|
|---|
| 701 | <tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>See <a class="reference external" href="http://www.syslinux.org/wiki/index.php?title=PXELINUX#Configuration">http://www.syslinux.org/wiki/index.php?title=PXELINUX#Configuration</a></td></tr>
|
|---|
| 702 | </tbody>
|
|---|
| 703 | </table>
|
|---|
| 704 | <table class="docutils footnote" frame="void" id="id8" rules="none">
|
|---|
| 705 | <colgroup><col class="label" /><col /></colgroup>
|
|---|
| 706 | <tbody valign="top">
|
|---|
| 707 | <tr><td class="label"><a class="fn-backref" href="#id3">[3]</a></td><td>See <a class="reference external" href="https://en.wikipedia.org/wiki/Aufs">https://en.wikipedia.org/wiki/Aufs</a></td></tr>
|
|---|
| 708 | </tbody>
|
|---|
| 709 | </table>
|
|---|
| 710 | <table class="docutils footnote" frame="void" id="id9" rules="none">
|
|---|
| 711 | <colgroup><col class="label" /><col /></colgroup>
|
|---|
| 712 | <tbody valign="top">
|
|---|
| 713 | <tr><td class="label"><a class="fn-backref" href="#id4">[4]</a></td><td>See <a class="reference external" href="http://shitwefoundout.com/wiki/Diskless_ubuntu">http://shitwefoundout.com/wiki/Diskless_ubuntu</a></td></tr>
|
|---|
| 714 | </tbody>
|
|---|
| 715 | </table>
|
|---|
| 716 | <table class="docutils footnote" frame="void" id="id10" rules="none">
|
|---|
| 717 | <colgroup><col class="label" /><col /></colgroup>
|
|---|
| 718 | <tbody valign="top">
|
|---|
| 719 | <tr><td class="label"><a class="fn-backref" href="#id5">[5]</a></td><td>See <a class="reference external" href="http://debianaddict.com/2012/06/19/diskless-debian-linux-booting-via-dhcppxenfstftp/">http://debianaddict.com/2012/06/19/diskless-debian-linux-booting-via-dhcppxenfstftp/</a></td></tr>
|
|---|
| 720 | </tbody>
|
|---|
| 721 | </table>
|
|---|
| 722 | <hr class="docutils" />
|
|---|
| 723 | <table class="docutils field-list" frame="void" rules="none">
|
|---|
| 724 | <col class="field-name" />
|
|---|
| 725 | <col class="field-body" />
|
|---|
| 726 | <tbody valign="top">
|
|---|
| 727 | <tr class="field"><th class="field-name">Status:</th><td class="field-body">$Id: TestBoxImaging.html 69111 2017-10-17 14:26:02Z vboxsync $</td>
|
|---|
| 728 | </tr>
|
|---|
| 729 | <tr class="field"><th class="field-name">Copyright:</th><td class="field-body">Copyright (C) 2010-2017 Oracle Corporation.</td>
|
|---|
| 730 | </tr>
|
|---|
| 731 | </tbody>
|
|---|
| 732 | </table>
|
|---|
| 733 | </div>
|
|---|
| 734 | </div>
|
|---|
| 735 | </body>
|
|---|
| 736 | </html>
|
|---|
