diff --git a/.github/ISSUE_TEMPLATE/hba_report.md b/.github/ISSUE_TEMPLATE/hba_report.md new file mode 100644 index 00000000..0884bfb3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/hba_report.md @@ -0,0 +1,15 @@ +--- +name: HBA report +about: Report a HBA that is working on the specific enviroment +assignees: piste-jp-ibm +lebels: HBA Report +title: HBA list request + +--- + +**Fill in the informatio of HBA and OS** + +| HBA type | Vendor | I/F type (FC, SAS etc) | OS | LTFS version or commit hash | Note | +| -------- | ------ | ---------------------- | --- | --------------------------- | ---- | +| | | | | | | + diff --git a/.github/workflows/build-centos7.yml b/.github/workflows/build-centos7.yml deleted file mode 100644 index f00d5e53..00000000 --- a/.github/workflows/build-centos7.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: CentOS7 Build Job - -on: [push, pull_request] - -jobs: - build: - name: Build on CentOS7 - runs-on: ubuntu-latest - - steps: - - name: Set up Git repository - uses: actions/checkout@v1 - - name: Build LTFS - id: build - uses: LinearTapeFileSystem/CentOS7-Build@v1.2 - with: - destination: '/tmp/ltfs' diff --git a/.github/workflows/build-centos8.yml b/.github/workflows/build-centos8.yml index 727896e1..c2080283 100644 --- a/.github/workflows/build-centos8.yml +++ b/.github/workflows/build-centos8.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: - name: Build on CentOS8 + name: Build on CentOS8 (Rocky Linux) runs-on: ubuntu-latest steps: @@ -12,6 +12,6 @@ jobs: uses: actions/checkout@v1 - name: Build LTFS id: build - uses: LinearTapeFileSystem/CentOS8-Build@v1.0 + uses: LinearTapeFileSystem/CentOS8-Build@v1.6 with: destination: '/tmp/ltfs' diff --git a/.github/workflows/build-debian10.yml b/.github/workflows/build-debian10.yml index b2a0a888..3ff0e4b7 100644 --- a/.github/workflows/build-debian10.yml +++ b/.github/workflows/build-debian10.yml @@ -12,6 +12,6 @@ jobs: uses: actions/checkout@v1 - name: Build LTFS id: build - uses: LinearTapeFileSystem/Debian10-Build@v1.0 + uses: LinearTapeFileSystem/Debian10-Build@v1.1 with: destination: '/tmp/ltfs' diff --git a/.github/workflows/build-ubuntu-eoan.yml b/.github/workflows/build-ubuntu-eoan.yml deleted file mode 100644 index 40f456b0..00000000 --- a/.github/workflows/build-ubuntu-eoan.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Ubuntu 19.10 Build Job - -on: [push, pull_request] - -jobs: - build: - name: Build on Ubuntu Eoan - runs-on: ubuntu-latest - - steps: - - name: Set up Git repository - uses: actions/checkout@v1 - - name: Build LTFS - id: build - uses: LinearTapeFileSystem/Ubuntu1910-Build@v1.0 - with: - destination: '/tmp/ltfs' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..d7aec110 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,76 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master, v2* ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '26 12 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'cpp', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + #- name: Autobuild + # uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + - run: | + sudo apt-get -q -y update + sudo apt-get -q -y upgrade + sudo apt-get -y install build-essential automake autoconf libtool pkg-config icu-devtools libicu-dev libxml2-dev uuid-dev fuse libfuse-dev libsnmp-dev + sudo cp .github/workflows/icu-config /usr/bin/icu-config + ./autogen.sh + ./configure + make + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/icu-config b/.github/workflows/icu-config new file mode 100755 index 00000000..d5aed413 --- /dev/null +++ b/.github/workflows/icu-config @@ -0,0 +1,12 @@ +#!/bin/sh + +opts=$1 + +case $opts in + '--cppflags') + echo '' ;; + '--ldflags') + echo '-licuuc -licudata' ;; + *) + echo '/usr/lib/x86_64-linux-gnu/icu/pkgdata.inc' ;; +esac diff --git a/.gitignore b/.gitignore index cae78d4d..0711b11e 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ ltfs.pc .libs/ messages/.lib .dirstamp +# Files created by OS +.DS_Store # Files for development GPATH GRTAGS diff --git a/.travis.yml b/.travis.yml index ac55a2f1..d4f7be5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,9 +33,9 @@ before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH="$PATH:$ICU_PATH:$LIBXML2_PATH" ; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install ossp-uuid gnu-sed ; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew cask install osxfuse ; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install --cask osxfuse ; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list ; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew cask list ; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list --cask ; fi - if [[ "$TRAVIS_DIST" == "xenial" ]]; then sudo apt-get update ; fi - if [[ "$TRAVIS_DIST" == "xenial" ]]; then sudo apt-get install -y libfuse2 libfuse-dev ; fi - if [[ "$TRAVIS_DIST" == "xenial" ]]; then sudo apt-get install -y libxml2 libxml2-dev ; fi @@ -50,7 +50,7 @@ before_install: - if [[ "$TRAVIS_DIST" == "bionic" ]]; then sudo apt-get install -y libuuid1 uuid-dev ; fi script: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./autogen.sh && LDFLAGS="-framework CoreFoundation -framework IOKit" ./configure --enable-icu-6x --disable-snmp && make ; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./autogen.sh && LDFLAGS="-framework CoreFoundation -framework IOKit" ./configure --enable-icu-6x --disable-snmp --enable-warning-as-error --enable-message-checker && make ; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./autogen.sh && ./configure --prefix=/tmp && make && make install ; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then cd /tmp && git clone https://github.com/LinearTapeFileSystem/ltfs-backends.git && cd ltfs-backends ; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/tmp/lib/pkgconfig ; fi diff --git a/README.md b/README.md index 21d978c7..59f03049 100644 --- a/README.md +++ b/README.md @@ -1,85 +1,18 @@ ![](https://img.shields.io/github/issues/lineartapefilesystem/ltfs.svg) ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/CentOS7%20Build%20Job/badge.svg?branch=master) -[![CodeFactor](https://www.codefactor.io/repository/github/lineartapefilesystem/ltfs/badge)](https://www.codefactor.io/repository/github/lineartapefilesystem/ltfs) [![BSD License](http://img.shields.io/badge/license-BSD-blue.svg?style=flat)](LICENSE) -# Linear Tape File System (LTFS) +# About this branch -Linear Tape File System (LTFS) is a filesystem to mount a LTFS formatted tape in a tape drive. Once LTFS mounts a LTFS formatted tape as filesystem, user can access to the tape via filesystem API. +This is the `master` branch of the LTFS project. At this time, this branch is used for version 2.5 development. So it wouldn't be stable a little. Please consider to follow the tree on `v2.4-stable` branch if you want to use stable codes. -Objective of this project is being the reference implementation of the LTFS format Specifications in [SNIA](https://www.snia.org/tech_activities/standards/curr_standards/ltfs). - -At this time, the target of this project to meet is the LTFS format specifications 2.4. (https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_2.4.0_TechPosition.pdf). +# What is the Linear Tape File System (LTFS) -## LTFS Format Specifications - -LTFS Format Specification is specified data placement, shape of index and names of extended attributes for LTFS. This specification is defined in [SNIA](https://www.snia.org/tech_activities/standards/curr_standards/ltfs) first and then it is forwarded to [ISO](https://www.iso.org/home.html) as ISO/IEC 20919 from version 2.2. +The Linear Tape File System (LTFS) is a filesystem to mount a LTFS formatted tape in a tape drive. Once LTFS mounts a LTFS formatted tape as filesystem, user can access to the tape via filesystem API. -The table below show status of the LTFS format Specification +Objective of this project is being the reference implementation of the LTFS format Specifications in [SNIA](https://www.snia.org/tech_activities/standards/curr_standards/ltfs). - | Version | Status of SNIA | Status of ISO | - |:-: |:-: |:-: | - | 2.2 | [Published the Technical Position](http://snia.org/sites/default/files/LTFS_Format_2.2.0_Technical_Position.pdf) | Published | - | 2.3.1 | [Published the Technical Position](https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_2.3.1_TechPosition.pdf) | - | - | 2.4 | [Published the Technical Position](https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_2.4.0_TechPosition.pdf) | On going | - | 2.5 | [Published the Technical Position](https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_v2.5_Technical_Position.pdf) | Not started yet | - -## How to use the LTFS (Quick start) - -This section is for person who already have a machine the LTFS is installed. - -Instruction how to use the LTFS is on [Wiki](https://github.com/LinearTapeFileSystem/ltfs/wiki). Please take a look! - -## Getting Started from GitHub project - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. - -## Prerequisites - -- Linux - * automake 1.13.4 or later - * autoconf 2.69 or later - * libtool 2.4.2 or later - * fuse 2.6.0 or later - * uuid 1.36 or later (Linux) - * libxml-2.0 2.6.16 or later - * net-snmp 5.3 or later - * icu4c 4.8 or later - -- OSX (macOS) - - Following packages on homebrew - - * automake - * autoconf - * libtool - * osxfuse (brew cask install osxfuse) - * ossp-uuid - * libxml2 - * icu4c - * gnu-sed - -- FreeBSD: - * FreeBSD 10.2 or 11.0 or later (for sa(4) driver changes) - * automake - * autoconf - * libtool - * fusefs-libs - * net-snmp - * e2fsprogs-libuuid - * libxml2 - * icu - -- NetBSD: - * NetBSD 7.0 or higher (for FUSE support) - * automake - * autoconf - * libtool - * libfuse - * net-snmp - * libuuid - * libxml2 - * icu +At this time, the target of this project to meet is the [LTFS format specifications 2.5](https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_v2.5_Technical_Position.pdf). ## Supported Tape Drives @@ -89,6 +22,7 @@ These instructions will get you a copy of the project up and running on your loc | IBM | LTO6 | None | | IBM | LTO7 | None | | IBM | LTO8 | HB81 | + | IBM | LTO9 | None | | IBM | TS1140 | 3694 | | IBM | TS1150 | None | | IBM | TS1155 | None | @@ -97,131 +31,136 @@ These instructions will get you a copy of the project up and running on your loc | HP | LTO6 | T.B.D. | | HP | LTO7 | T.B.D. | | HP | LTO8 | T.B.D. | + | HP | LTO9 | T.B.D. | | Quantum | LTO5 (Only Half Height) | T.B.D. | | Quantum | LTO6 (Only Half Height) | T.B.D. | | Quantum | LTO7 (Only Half Height) | T.B.D. | | Quantum | LTO8 (Only Half Height) | T.B.D. | + | Quantum | LTO9 (Only Half Height) | T.B.D. | -## Installing +## LTFS Format Specifications -### Build and install on Linux +LTFS Format Specification is specified data placement, shape of index and names of extended attributes for LTFS. This specification is defined in [SNIA](https://www.snia.org/tech_activities/standards/curr_standards/ltfs) first and then it is forwarded to [ISO](https://www.iso.org/home.html) as ISO/IEC 20919 from version 2.2. -``` -./autogen.sh -./configure -make -make install -``` +The table below show status of the LTFS format Specification -`./configure --help` shows various options for build and install. + | Version | Status of SNIA | Status of ISO | + |:-------:|:---------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------:| + | 2.2 | [Published](http://snia.org/sites/default/files/LTFS_Format_2.2.0_Technical_Position.pdf) | [Published as `20919:2016`](https://www.iso.org/standard/69458.html) | + | 2.3.1 | [Published](https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_2.3.1_TechPosition.PDF) | - | + | 2.4 | [Published](https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_2.4.0_TechPosition.pdf) | - | + | 2.5.1 | [Published](https://www.snia.org/sites/default/files/technical-work/ltfs/release/SNIA-LTFS-Format-2-5-1-Standard.pdf) | [Published as `20919:2021`](https://www.iso.org/standard/80598.html) | -#### Parameter settings of the sg driver +# How to use the LTFS (Quick start) -LTFS uses the sg driver by default. You can improve reliability to change parameters of the sg driver below. +This section is for a person who already has a machine with the LTFS installed. Instructions on how to use the LTFS is also available on [Wiki](https://github.com/LinearTapeFileSystem/ltfs/wiki). -``` -def_reserved_size=1048576 -``` +## Step1: List tape drives -In RHEL7, you can put following file as `/etc/modprobe.d/sg.conf`. +`# ltfs -o device_list` + +The output is as follows. You have 3 drives in this example and you can use "Device Name" field, like `/dev/sg43` in this case, as the argument of ltfs command to mount the tape drive. ``` -options sg def_reserved_size=1048576 +50c4 LTFS14000I LTFS starting, LTFS version 2.4.0.0 (10022), log level 2. +50c4 LTFS14058I LTFS Format Specification version 2.4.0. +50c4 LTFS14104I Launched by "/home/piste/ltfsoss/bin/ltfs -o device_list". +50c4 LTFS14105I This binary is built for Linux (x86_64). +50c4 LTFS14106I GCC version is 4.8.5 20150623 (Red Hat 4.8.5-11). +50c4 LTFS17087I Kernel version: Linux version 3.10.0-514.10.2.el7.x86_64 (mockbuild@x86-039.build.eng.bos.redhat.com) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-11) (GCC) ) #1 SMP Mon Feb 20 02:37:52 EST 2017 i386. +50c4 LTFS17089I Distribution: NAME="Red Hat Enterprise Linux Server". +50c4 LTFS17089I Distribution: Red Hat Enterprise Linux Server release 7.3 (Maipo). +50c4 LTFS17089I Distribution: Red Hat Enterprise Linux Server release 7.3 (Maipo). +50c4 LTFS17085I Plugin: Loading "sg" tape backend. +Tape Device list:. +Device Name = /dev/sg43, Vender ID = IBM , Product ID = ULTRIUM-TD5 , Serial Number = 9A700L0077, Product Name = [ULTRIUM-TD5] . +Device Name = /dev/sg38, Vender ID = IBM , Product ID = ULT3580-TD6 , Serial Number = 00013B0119, Product Name = [ULT3580-TD6] . +Device Name = /dev/sg37, Vender ID = IBM , Product ID = ULT3580-TD7 , Serial Number = 00078D00C2, Product Name = [ULT3580-TD7] . ``` -You can check current configuration of sg driver to see the file `/proc/scsi/sg/debug` like +## Step2: Format a tape -``` -$ cat /proc/scsi/sg/debug -max_active_device=44 def_reserved_size=32768 - >>> device=sg25 1:0:10:0 em=0 sg_tablesize=1024 excl=0 open_cnt=1 - FD(1): timeout=60000ms bufflen=524288 (res)sgat=16 low_dma=0 - cmd_q=1 f_packid=0 k_orphan=0 closed=0 - No requests active - >>> device=sg26 1:0:10:1 em=0 sg_tablesize=1024 excl=0 open_cnt=1 - FD(1): timeout=60000ms bufflen=524288 (res)sgat=16 low_dma=0 - cmd_q=1 f_packid=0 k_orphan=0 closed=0 - No requests active -``` +As described in the LTFS format specifications, LTFS uses the partition feature of the tape drive. This means you can't use a tape just after you purchase a tape. You need format the tape before using it on LTFS. -##### Performance improvement of the sg device +To format a tape, you can use `mkltfs` command like -You can improve performance to change parameters of the sg driver below. But this option may cause I/O error reported as #144 in some HBAs. +`# mkltfs -d 9A700L0077` -``` -allow_dio=1 -``` +In this case, `mkltfs` tries to format a tape in the tape drive `9A700L0077`. You can use the device name `/dev/sg43` instead. -In RHEL7, you can put following file as `/etc/modprobe.d/sg.conf`. +## Step3: Mount a tape through a tape drive -``` -options sg allow_dio=1 -``` +After you prepared a formatted tape, you can mount it through a tape drive like -At this time, we know following HBA's works correctly. +`# ltfs -o devname=9A700L0077 /ltfs` -- QLogic 8Gb FC HBAs +In this command, the ltfs command will try to mount the tape in the tape drive `9A700L0077` to `/ltfs` directory. Of course, you can use a device name `/dev/sg43` instead. -And following HBA's doesn't work correctly. +If the mount process is successfully done, you can access to the LTFS tape through `/ltfs` directory. -- ATTO ExpressSAS H680 -- Emulex FC HBAs (Some drivers work but some drivers dont work, see this [section](#note-for-the-lpfc-driver-emulex-fibre-hbas)) +You must not touch any `st` devices while ltfs is mounting a tape. -##### Note for the lpfc driver (Emulex Fibre HBAs) +## Step4: Unmount the tape drive -In the lpfc driver (for Emulex Fibre HBAs), the table size of the scatter-gather is 64 by default. This configuration may cause I/O errors intermittently when `allow_dio=1` is set and scatter-gather table cannot be reserved. To avoid this error, you need to change the parameter `lpfc_sg_seg_cnt` to 256 or greater like below. +You can use following command when you want to unmount the tape. The ltfs command try to write the current meta-data to the tape and close the tape cleanly. -``` -options lpfc lpfc_sg_seg_cnt=256 -``` +`# umount /ltfs` -In some versions of the lpfc driver (for Emulex Fibre HBAs), the table size of the scatter-gather cannot be changed correctly. You can check the value is changed or not in `sg_tablesize` value in `/proc/scsi/sg/debug`. If you don't have a correct value (256 or greater) in `sg_tablesize`, removing `allow_dio=1` configuration of the sg driver is strongly recommended. +One thing you need to pay attention to here is, that the unmount command continues to work in the background after it returns. It just initiates a trigger to notify the the ltfs command of the unmount request. Actual unmount is completed when the ltfs command is finished. -##### Note for buggy HBAs +## The `ltfsee_ordered_copy` utility -LTFS doesn't support the HBAs which doesn't handle the transfer length of SCSI data by default. The reason is because the safety of the data but LTFS provides a option to relax this limitation. +The [`ltfsee_ordered_copy`](https://github.com/LinearTapeFileSystem/ltfs/wiki/ltfs_ordered_copy) is a program to copy files from source to destination with LTFS order optimization. -You can use such kind of HBAs if you run the `configure` script with `--enable-buggy-ifs` option and build. +It is written in python and it can work with both python2 and python3 (Python 2.7 or later is strongly recommended). You need to install the `pyxattr` module for both python2 and python3. -List of the HBA `--enable-buggy-ifs` is needed is below. +# Building the LTFS from this GitHub project -[HBA list require `--enable-buggy-ifs`](https://github.com/LinearTapeFileSystem/ltfs/wiki/HBA-info) +These instructions will get a copy of the project up and running on your local machine for development and testing purposes. -#### IBM lin_tape driver support +## Prerequisites for build -You need to add `--enable-lintape` as an argument of ./configure script if you want to build the backend for lin_tape. You also need to add `DEFAULT_TAPE=lin_tape` if you set the lin_tape backend as default backend. +Please refer [this page](https://github.com/LinearTapeFileSystem/ltfs/wiki/Build-Environments). -#### Buildable distributions +## Build and install on Linux + +``` +./autogen.sh +./configure +make +make install +``` + +`./configure --help` shows various options for build and install. - | Dist | Arch | Status | - |:-: |:-: |:-: | - | RHEL 8 | x86_64 | OK | - | RHEL 8 | ppc64le | OK | - | RHEL 7 | x86_64 | OK | - | RHEL 7 | ppc64le | OK | - | CentOS 8 | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/CentOS8%20Build%20Job/badge.svg?branch=master)| - | CentOS 8 | ppc64le | Probably OK | - | CentOS 7 | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/CentOS7%20Build%20Job/badge.svg?branch=master)| - | CentOS 7 | ppc64le | Probably OK | - | Fedora 28 | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Fedora28%20Build%20Job/badge.svg?branch=master)| - | Ubuntu 16.04 LTS | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Ubuntu%2016.04%20Build%20Job/badge.svg?branch=master)| - | Ubuntu 16.04 LTS | ppc64le | [![Build Status](https://travis-ci.org/LinearTapeFileSystem/ltfs.svg?branch=master)](https://travis-ci.org/LinearTapeFileSystem/ltfs) | - | Ubuntu 18.04 LTS | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Ubuntu%2018.04%20Build%20Job/badge.svg?branch=master)| - | Ubuntu 18.04 LTS | ppc64le | [![Build Status](https://travis-ci.org/LinearTapeFileSystem/ltfs.svg?branch=master)](https://travis-ci.org/LinearTapeFileSystem/ltfs) | - | Ubuntu 19.10 (Need icu-config) | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Ubuntu%2019.10%20Build%20Job/badge.svg?branch=master)| - | Ubuntu 20.04 LTS (Need icu-config)| x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Ubuntu%2020.04%20Build%20Job/badge.svg?branch=master)| - | Debian 9 | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Debian9%20Build%20Job/badge.svg?branch=master)| - | Debian 10 (Need icu-config) | x86_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Debian10%20Build%20Job/badge.svg?branch=master)| - | ArchLinux 2018.08.01 | x86_64 | OK | - | ArchLinux 2018.12.31 (rolling) | x86_64 | OK | +In some systems, you might need `sudo ldconfig -v` after `make install` to load the shared libraries correctly. + +### Buildable Linux distributions + + | Dist | Arch | Status | + |:----------------------------------:|:-------:|:--------------------------------------------------------------------------------------------------------------------------------:| + | RHEL 8 | x86\_64 | OK - Not checked automatically | + | RHEL 8 | ppc64le | OK - Not checked automatically | + | CentOS 8 (Rocky Linux) | x86\_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/CentOS8%20Build%20Job/badge.svg?branch=master) | + | CentOS 8 (Rocky Linux) | ppc64le | OK - Not checked automatically | + | Fedora 28 | x86\_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Fedora28%20Build%20Job/badge.svg?branch=master) | + | Ubuntu 16.04 LTS | x86\_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Ubuntu%2016.04%20Build%20Job/badge.svg?branch=master) | + | Ubuntu 16.04 LTS | ppc64le | OK - Not checked automatically | + | Ubuntu 18.04 LTS | x86\_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Ubuntu%2018.04%20Build%20Job/badge.svg?branch=master) | + | Ubuntu 18.04 LTS | ppc64le | OK - Not checked automatically | + | Ubuntu 20.04 LTS (Need icu-config) | x86\_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Ubuntu%2020.04%20Build%20Job/badge.svg?branch=master) | + | Debian 9 | x86\_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Debian9%20Build%20Job/badge.svg?branch=master) | + | Debian 10 (Need icu-config) | x86\_64 | ![GH Action status](https://github.com/LinearTapeFileSystem/ltfs/workflows/Debian10%20Build%20Job/badge.svg?branch=master) | + | ArchLinux 2018.08.01 | x86\_64 | OK - Not checked automatically | + | ArchLinux 2018.12.31 (rolling) | x86\_64 | OK - Not checked automatically | Currently, automatic build checking is working on GitHub Actions and Travis CI. -For Ubuntu19.10, Ubuntu20.04 and Debian10, dummy `icu-config` is needed in the build machine. See Issue [#153](https://github.com/LinearTapeFileSystem/ltfs/issues/153). +For Ubuntu20.04 and Debian10, dummy `icu-config` is needed in the build machine. See Issue [#153](https://github.com/LinearTapeFileSystem/ltfs/issues/153). -### Build and install on OSX (macOS) +## Build and install on OSX (macOS) -#### Recent Homedrew system setup +### Recent Homebrew system setup Before build on macOS, you need to configure the environment like below. @@ -232,7 +171,7 @@ export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:/usr/local/opt/libxml export PATH="$PATH:$ICU_PATH:$LIBXML2_PATH" ``` -#### Old Homedrew system setup +### Old Homebrew system setup Before build on OSX (macOS), some include path adjustment is required. ``` @@ -240,7 +179,7 @@ brew link --force icu4c brew link --force libxml2 ``` -#### Building LTFS +### Building LTFS On OSX (macOS), snmp cannot be supported, you need to disable it on configure script. And may be, you need to specify LDFLAGS while running configure script to link some required frameworks, CoreFundation and IOKit. ``` @@ -252,13 +191,15 @@ make install `./configure --help` shows various options for build and install. -#### Buildable systems +#### Buildable macOS systems - | OS | Xcode | Package system | Status | - |:-: |:-: |:-: |:-: | - | macOS 10.14.6 | 11.3 | Homebrew | [![Build Status](https://travis-ci.org/LinearTapeFileSystem/ltfs.svg?branch=master)](https://travis-ci.org/LinearTapeFileSystem/ltfs) | + | OS | Xcode | Package system | Status | + |:-------------:|:-----:|:--------------:|:-------------------------------------------------------------------------------------------------------------------------------------:| + | macOS 10.14.6 | 11.3 | Homebrew | OK - Not checked automatically | + | macOS 10.15 | 12.4 | Homebrew | OK - Not checked automatically | + | macOS 11 | 12.4 | Homebrew | OK - Not checked automatically | -### Build and install on FreeBSD +## Build and install on FreeBSD Note that on FreeBSD, the usual 3rd party man directory is /usr/local/man. Configure defaults to using /usr/local/share/man. So, override it on the command line to avoid having man pages put in the wrong place. @@ -271,9 +212,10 @@ make install #### Buildable versions - | Version | Arch | Status | - |:-: |:-: |:-: | - | 11 | x86_64 | OK | + | Version | Arch | Status | + |:-------:|:-------:|:------------------------------:| + | 11 | x86\_64 | OK - Not checked automatically | + | 12 | x86\_64 | OK - Not checked automatically | ### Build and install on NetBSD @@ -286,11 +228,11 @@ make install #### Buildable versions - | Version | Arch | Status | - |:-: |:-: |:-: | - | 8.1 | amd64 | OK | - | 8.0 | i386 | OK | - | 7.2 | amd64 | OK | + | Version | Arch | Status | + |:-------:|:-----:|:------------------------------:| + | 8.1 | amd64 | OK - Not checked automatically | + | 8.0 | i386 | OK - Not checked automatically | + | 7.2 | amd64 | OK - Not checked automatically | ## Contributing diff --git a/autogen.sh b/autogen.sh index 504a9748..113fb3f2 100755 --- a/autogen.sh +++ b/autogen.sh @@ -46,6 +46,6 @@ else libtoolize fi -autoconf -autoheader -automake --add-missing --copy +autoconf || echo "Ignore warning" +autoheader || echo "Ignore warning" +automake --add-missing --copy || echo "Ignore warning" diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..2b967245 --- /dev/null +++ b/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# This is the script for building the tree under CodeQL scanning in github flow. +# Do not use on other environments + +sudo apt-get -q -y update +sudo apt-get -q -y upgrade +sudo apt-get -y install build-essential automake autoconf libtool pkg-config icu-devtools libicu-dev libxml2-dev uuid-dev fuse libfuse-dev libsnmp-dev +./autogen.sh +./configure +make diff --git a/configure.ac b/configure.ac index 8ad4062d..369f66ae 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ ;; OO_Copyright_BEGIN ;; ;; -;; Copyright 2010, 2020 IBM Corp. All rights reserved. +;; Copyright 2010, 2021 IBM Corp. All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions @@ -36,11 +36,10 @@ dnl dnl LTFS configure.ac. dnl -AC_INIT([LTFS], [2.4.3.1 (Prelim)], IBM corporation.) +AC_INIT([LTFS], [2.5.0.0 (Prelim)], IBM corporation.) AC_CONFIG_SRCDIR([src/main.c]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIRS([m4]) -AC_CONFIG_MACRO_DIR([m4]) AC_CANONICAL_HOST AC_CANONICAL_BUILD AC_CANONICAL_TARGET @@ -50,7 +49,7 @@ AC_USE_SYSTEM_EXTENSIONS AC_PROG_CC_C99 AM_PROG_CC_C_O AM_PROG_AR -AC_PROG_LIBTOOL +LT_INIT dnl dnl Detecting OS @@ -153,22 +152,13 @@ AC_ARG_ENABLE([snmp], AC_MSG_RESULT([$snmp]) dnl -dnl Handle --enable-new-locking (default: depend on platform) -dnl NetBSD: old locking is broken, force new locking -dnl others: default to old locking +dnl Handle --enable-new-locking (default: yes) dnl -if test "x${host_netbsd}" = "xyes" -then - use_new_locking_default=yes -else - use_new_locking_default=no -fi - AC_MSG_CHECKING([whether to enable new locking system or not]) AC_ARG_ENABLE([new-locking], [AS_HELP_STRING([--enable-new-locking],[Use new locking system or not])], [use_new_locking=$enableval], - [use_new_locking=$use_new_locking_default] + [use_new_locking=yes] ) AC_MSG_RESULT([$use_new_locking]) @@ -194,6 +184,28 @@ AC_ARG_ENABLE([buggy_ifs], ) AC_MSG_RESULT([$buggy_ifs]) +dnl +dnl Handle --enable-xml-indent (default:no) +dnl +AC_MSG_CHECKING([whether to enable xml indentation for index]) +AC_ARG_ENABLE([xml_indent], + [AS_HELP_STRING([--enable-xml-indent],[Enable XML indentation for index])], + [xml_indent=$enableval], + [xml_indent=no] +) +AC_MSG_RESULT([$xml_indent]) + +dnl +dnl Handle --enable-format-spec25 (default:no) +dnl +AC_MSG_CHECKING([whether to enable format spec 2.5 support or not]) +AC_ARG_ENABLE([format_spec25], + [AS_HELP_STRING([--enable-format-spec25],[Support format spec 2.5])], + [format_spec25=$enableval], + [format_spec25=no] +) +AC_MSG_RESULT([$format_spec25]) + dnl dnl Handle extra compile flags for tape driver dnl @@ -338,16 +350,20 @@ fi dnl dnl Check for ICU dnl -ICU_MODULE_CFLAGS="`icu-config --cppflags 2> /dev/null`"; -ICU_MODULE_LIBS="`icu-config --ldflags 2> /dev/null`"; -if test -z "$ICU_MODULE_LIBS" +AC_MSG_CHECKING(ICU version) +ICU_MODULE_VERSION="`pkg-config --modversion icu-i18n 2> /dev/null`"; +ICU_MODULE_CFLAGS="`pkg-config --cflags icu-i18n 2> /dev/null`"; +ICU_MODULE_LIBS="`pkg-config --libs icu-i18n 2> /dev/null`"; +AC_MSG_RESULT([$ICU_MODULE_VERSION]) + +if test -z "$ICU_MODULE_VERSION" then - PKG_CHECK_MODULES([ICU_MODULE], [icu >= 0.21]) + PKG_CHECK_MODULES([ICU_MODULE], [icu-uc >= 0.21]) fi -AC_MSG_CHECKING([use latest ICU]) +AC_MSG_CHECKING([for using ICU6x (unorm2) functions forcibly]) AC_ARG_ENABLE([icu_6x], - [AS_HELP_STRING([--enable-icu-6x],[Support handling of ICU 6x functions])], + [AS_HELP_STRING([--enable-icu-6x],[Force to use ICU6x (unorm2) functions])], [icu_6x=$enableval], [icu_6x=no] ) @@ -355,13 +371,16 @@ AC_MSG_RESULT([$icu_6x]) if test "x${icu_6x}" = "xyes" then - AC_MSG_CHECKING(for ICU version) - ICU_MODULE_VERSION="`icu-config --version 2> /dev/null`"; - if test "${ICU_MODULE_VERSION%%.*}" -ge "60" + AM_EXTRA_CPPFLAGS="${AM_EXTRA_CPPFLAGS} -D USE_UNORM2" +else + AC_MSG_CHECKING(for using unorm2 functions) + if test "${ICU_MODULE_VERSION%%.*}" -ge "56" then - AM_EXTRA_CPPFLAGS="${AM_EXTRA_CPPFLAGS} -D ICU6x" + AM_EXTRA_CPPFLAGS="${AM_EXTRA_CPPFLAGS} -D USE_UNORM2" + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT(no) fi - AC_MSG_RESULT([$ICU_MODULE_VERSION]) fi dnl @@ -375,7 +394,7 @@ if test "x${snmp}" != "xno" then SNMP_ENABLE="-D ENABLE_SNMP" SNMP_MODULE_CFLAGS="`net-snmp-config --cflags 2> /dev/null`"; - SNMP_MODULE_LIBS_A="`net-snmp-config --agent-libs` 2> /dev/null"; + SNMP_MODULE_LIBS_A="`net-snmp-config --agent-libs 2> /dev/null`"; SNMP_MODULE_LIBS="`net-snmp-config --libs 2> /dev/null`"; if test -z "$SNMP_MODULE_LIBS_A" then @@ -391,7 +410,7 @@ fi dnl dnl Check for headers, types, structures, compiler characteristics dnl -AC_CHECK_HEADERS([fcntl.h limits.h stddef.h stdint.h stdlib.h string.h sys/ioctl.h sys/mount.h sys/time.h termios.h unistd.h]) +AC_CHECK_HEADERS([fcntl.h limits.h stddef.h stdint.h stdlib.h string.h sys/ioctl.h sys/mount.h sys/time.h termios.h unistd.h sys/sysctl.h]) AC_HEADER_STDBOOL AC_TYPE_MODE_T AC_TYPE_OFF_T @@ -430,11 +449,12 @@ SAVE_CFLAGS=$CFLAGS CFLAGS="$CFLAGS ${LIBXML2_MODULE_CFLAGS}" AC_MSG_CHECKING(for XML_PARSE_HUGE) AC_CACHE_VAL(ac_cv_xml_huge, - AC_TRY_COMPILE([#include ], - [int foo = XML_PARSE_HUGE; return 0], - ac_cv_xml_huge=yes, - ac_cv_xml_huge=no - )) + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([#include ], + [int foo = XML_PARSE_HUGE; return 0;])], + [ac_cv_xml_huge=yes], + [ac_cv_xml_huge=no] + )) AC_MSG_RESULT($ac_cv_xml_huge) CFLAGS="$SAVE_CFLAGS" @@ -501,6 +521,16 @@ then AM_CPPFLAGS="${AM_CPPFLAGS} -DPOSIXLINK_ONLY" fi +if test "x$xml_indent" = "xyes" +then + AM_CPPFLAGS="${AM_CPPFLAGS} -DINDENT_INDEXES" +fi + +if test "x$format_spec25" = "xyes" +then + AM_CPPFLAGS="${AM_CPPFLAGS} -DFORMAT_SPEC25" +fi + dnl dnl Specify CPU specific optimizer options for CRC calculation dnl @@ -519,6 +549,11 @@ then SSE42=yes fi + if test ${GCC_VERSION_MAJOR} -ge 8 -a ${GCC_VERSION_MINOR} -ge 0 -a "x$warning_as_error" = "xyes" + then + AM_CFLAGS="${AM_CFLAGS} -Wno-stringop-truncation" + fi + if test "x${SSE42}" = "xyes" then case x"$target_cpu" in @@ -569,6 +604,7 @@ fi if test "x${host_mac}" = "xno" then MODULE_CHECK_LD='-Wl,--no-undefined,--as-needed' + AM_LDFLAGS="${AM_LDFLAGS} -ldl" fi AM_CFLAGS=`echo ${AM_CFLAGS} | ${SED} 's|-D_FORTIFY_SOURCE=. ||g'` diff --git a/contrib/fssim/README.md b/contrib/fssim/README.md new file mode 100644 index 00000000..c4550c18 --- /dev/null +++ b/contrib/fssim/README.md @@ -0,0 +1,10 @@ +# LTFS File System Simulator (fssim) + +The major change in version 2.5 of the [LTFS Format Specification](https://www.snia.org/sites/default/files/technical_work/LTFS/LTFS_Format_v2.5_Technical_Position.pdf) is the addition of Incremental Indexes. Incremental Indexes allow an LTFS index written to the Data Partition of the tape to contain only the changes from the prior index, rather than the full state of the file system. For a full discussion of Incremental Indexes, see Appendix H of the Format Specification referenced above. + +When we first proposed Incremental Indexes there was considerable discussion concerning what they would need to contain, and how they would be implemented. I decided to write a simulator that would mimic file system operations, and create full and incremental XML indexes very similar to those in LTFS as a Proof of Concept. This simulator also includes programs to merge full and incremental directories created by it and verify the results, as well as a directory of test cases built by various members of the LTFS project. + +The programs are meant to demonstrate how Incremental Indexes work, and also to potentially serve as an example of how they could be implemented. + +Full documentation of the programs is contained in the README file in the docs directory. + diff --git a/contrib/fssim/docs/README.md b/contrib/fssim/docs/README.md new file mode 100644 index 00000000..06300c54 --- /dev/null +++ b/contrib/fssim/docs/README.md @@ -0,0 +1,381 @@ + +# LTFS File System Simulator (fssim) +# Usage Guide and Examples + + +## Overview + +The File System Simulator programs are used to simulate file system activity, +create full and incrmental indexes reflecting that activity, update a full +index using incremental indexes, and verify the correctness of that update. + +The first program in the set is _fssim_. It simulates file system activity such +as creating or deleting files and directories, moving and copying files or +directories, listing information, etc. It also writes full and incremental +XML indexes to text files on request. + +The second program is _fsidxupd_ (perhaps a poor choice of names). It updates +a full index created by fssim with one or more incremental indexes created +sequentially in the same execution of fssim, then creates a full index +from the result. The resulting full index should exactly match the +corresponding full index from fssim. + +The programs were written by David Pease at IBM's Almaden Research Center in +May-August of 2018, and significantly revised and improved by the original +author (now retired) in September of 2020. + +## Invoking the programs: + +The programs are python executables, meaning that they can be invoked from +the command line in Unix-like environments (such as Linux and Mac) as long as +the executable bit is on and /usr/bin/python points to the python interpreter. The programs are written so that they run correctly in either python 2 or +python 3. + +In Windows, it should be possible to invoke them by a command such as "python +fsXXX" or by renaming the two program files to fsXXX.py and invoking them by +name (if python is properly configured). + +There are two additional files besides the _fssim_ and _fsidxupd_ programs. They +are _fscommon.py_ and _fsglobals.py_. Both of these are imported by the two +command programs; all four files should reside in the same directory. + +### fssim + +The format of the _fssim_ command is: + + fssim [-d] [command file] + +where: +* -d: prints a ton of debugging messages (probably meaningfull only to me) during processing +* command file: executes commands from the specified text file at startup + + A command file is simply a text file containing commands as they would be + entered from the keyboard; blank lines and comment lines starting with "#" + are allowed. Command files make it easy to design and debug a scenario, + then execute it multiple times (often while modifying code + in-between). After a command file is executed (without encountering an + "exit" command), the program drops into its normal keyboard input mode + (which is terminated by entering "exit"). + +### fsidxupd + +The format of the _fsidxupd_ command is: + + fsidxupd [-d] [ ...] + +where: +* -d: prints lots of debugging messages during processing +* fullindex: a file containing a full index from fssim +* incrindexes:a list of files containing incremental indexes from fssim + +The first file name must identify a full index created by _fssim_. The second +and subsequent names, if present, must identify incremental indexes created +sequentially by _fssim_. The output of the program (currently) has a hard-coded +file name of _upd-full.xml_. + +## fssim Commands + +The _fssim_ commands are closely modeled after the Unix shell commands that +perform the same function; however, they are often greatly simplified versions +(the goal was to implement a simulator, not re-write bash or busybox). + +Unrecognized commands or commands with an invalid number of operands will +return an **"\*\* Invalid"** message. Commands that do not execute successfully +(for any reason) will return the message **"\*\* Failed"**. That is the extent +of error messages. + +The command prompt indicates the current working directory (as it typically +does in bash), and starts at root ("/"). File and directory names can be +entered as full paths or relative to the current directory; however, the only +place that ".." is recognized is as the single operand of a _cd_ command, and +the only place that "." is recognized is as the single target for a _mv_ or _cp_ command. The directory separator character is "/". The command prompt +environment supports command recall and editing using the _readline_ library (tested on Linux and Mac). + +The command processor respects strings enclosed in double quotes ("). Quoted +strings can be used to create object names with embedded blanks or +to echo strings with arbitrary blank spaces. For example: + + echo Hello > "file 1"` + echo " world!" >> "file 1"` + cat "file 1"` + +The _help_ command will print a short description of all of the commands. +Here is a slightly more detailed command summary (square brackets indicate +optional parameters, curly braces indicate a choice): + +|name|operands|description| +|:-|:-|:-| +|cat|name|write a file's contents to the screen| +|cd|[dirname]|change current directory; with no operands, changes to root| +|cls||clear the screen (tested in Linux terminal window)| +|cp|[-r] name name|copy a file from one name to another; with -r option, copy directory contents recursively| +|dir|[-a] [name]|an alias for ls| +|echo|data [{>\|>>} name]|write data to the screen or to a file| +|fsck||verify that all object flags have been correctly reset | +|index|[{-i\|-f}] fileprefix|write full and/or incremental indexes to files using names prefixed with the specified prefix (see below)| +|ll|[name]|list long: an alias for ls -a | +|log|[-t]|list current update log entries (optionally in time order)| +|ls|[-a] [name]|list file or directory info (see more info below)| +|md|dirname [...]|an alias for mkdir| +|mkdir|dirname[...]|create directories or full directory paths| +|mv|name name|move/rename a file or directory| +|rd|dirname|an alias for rmdir| +|rm|filename|remove a file| +|rmdir|dirname|remove a directory (if it is empty)| +|touch|name [...] |update the timestamp of existing files or directories, or create new files (with directory paths if needed)| +|tree|[-o]|show a tree view of the entire file system; with -o option, also include oids| + +The _ls_ (or _dir_) command prefixes each file or directory with some attributes in +the order **"dmn"**. A **"d"** in the first position indicates a directory. An **"m"** +in the second position indicates that the file or directory has been modified +since the last index was written. An **"n"** in the third position indicates that +the directory is new since the last index (and thus must be written in its entirety with all of its children to an incremental index). Following the +attributes are the object id, the modification time, and the object size. For +files, the size is the number of data bytes in the file; for directories, it +is the number of children in the directory. When the object of the _ls_ +command is a directory, the "-a" (all) option will also show an entry for "." +which lists the status of the directory itself. (The _ll_ command is shorthand +for _ls -a_.) + +The _tree_ command shows a full file system tree view; using the "-o" option +shows the object ids (oids) following the object names, as in this example: + + / (0) + |-file1 (1) + |=dir1 (2) + |=dir2 (4) + |-file2 (5) + |=dir3 (3) + +As can be seen from the example, a "-" in front of a name in the tree view +indicates a file, while "=" indicates a directory. Within a directory +files are listed first followed by subdirectories, in alphabetical order. + +The _mkdir_ (or _md_), _touch_, and _echo_ commands will create any missing lower-level +directories required in the directory or file path. + +A note using on the _cp_ command with the "-r" option (recursive copy). This +command option requires two existing directory names as input; it copies the +**contents of the first directory to the second, recursively. Thus, using a +directory structure such as that shown in the output of _tree_ above, the +following command: + + cp -r dir1/dir2 dir3 + +would copy only file2 to dir3. In order to end up with dir2 and file2 under +dir3, one could use either + + cp -r dir1 dir3 + +or + + md dir3/dir2 + cp -r dir1/dir2 dir3/dir2 + +Note, however, that the first option will copy everything under dir1 to dir3; +if dir1 had files or directories other than dir2 under it, it would copy +everything and would not give the desired results. + +## Indexes + +By default, the _index_ command writes both a full index and an increment index +to a pair of files whose names start with the specified prefix. For example, +the command _index t0_ will create files _t0-full.xml_ and _t0-incr.xml_ in +the user's current directory. (The rationale for writing both indices is that +that one can start from any full index written, apply any number of subsequent +incremental indexes they wish, and the result should be the same as the full +index corresponding to the last incremental index used.) The _index_ command +allow overriding the default behavior by using the "-i" (incremental index +only) or "-f" (full index only) options. To simulate normal system operation, +(and minimize the number of files produced during testing), one could specify +"-f" for the index produced at the start of the run (e.g., _index -f t0_), then +use "-i" for all subsequent indexes up to the last one, and use neither flag +for the final index. This would give a starting and ending full index, and a +full set of incremental indexes to validate against the final full index. + +When testing indexes it's probably a good idea to start a command file with +the command _index [-f] t0_ (or the prefix of your choice); this creates a +starting point for incremental indexes from the beginning of the session. _index_ commands can be interspersed in a command stream at any point, and the +resulting full and incremental indexes can be inspected (and processed using _fsidxupd_). + +The _fsck_ command searches the entire file system for any files or directories +with **New** or **Modified** flags. This command can be used after an _index_ command with the +"-i" (incremental only) flag to verify that all objects with these flags have +been visited, and that all such flags have been correctly reset. + +A pair of "t0" indexes (created at startup) looks like this: + + + 0 + + + + + + + 0 + + + + + +The two are essentially identical, since the root directory is the only one in +existence, and it is new and thus shows up in the incremental index as well. (Actually, the t0 incremental index is useless, but it helps illustrate how +indexes work.) + +Running _fssim_ with this set of commands (e.g., in a command file): + + index t0 + mkdir Dir1 + touch Dir2/Dir3/foo + touch Dir2/Dir3/bar + index t1 + touch Dir2/Dir3/baz + rmdir Dir1 + index t2 + +produced the following two "t2" indexes: + + + 0 + + + + Dir2 + 2 + + + + Dir3 + 3 + + + + 5 + bar + + + + 6 + baz + + + + 4 + foo + + + + + + + + + + + + + + Dir1 + + + + Dir2 + + + Dir3 + + + + baz + 6 + + + + + + + + + +## Using fsidxupd + +_fsidxupd_ is used to verify the correctness of incremental indexes and their +application to a full index. For example, using the sample list of commands +in the Index section above, three pairs of indexes will be created by fssim +when running the commands; the file names will be _t0-full.xml, t0-incr.xml, +t1-full.xml, t1-incr.xml, t2-full.xml_, and _t2-incr.xml_. The following +_fsidxupd_ command will take the starting full index and apply the two appropriate incremental indexes: + + fsidxupd t0-full.xml t1-incr.xml t2-incr.xml + +The output of this command will be a file named _upd-full.xml_. In this example, +if the system is working correctly _upd-full.xml_ will be identical to +_t2-full.xml_. In Unix-like systems, the command: + + diff upd-full.xml t2-full.xml + +would show no output (no differences). + +## Program Organization + +The file _fscommon.py_ contains all of the file system logic. It contains +object definitions for directory and file objects, and implements all of +the low-level logic for managing those objects. (In function, it roughly +corresponds to the vfs module in a Unix-like OS.) It also contains the +the python version-agnostic print function (_fsprt_). + +The file _fsglobals.py_ defines global variables that must be shared between +the different modules. In order to provide a consistent naming reference +to these shared global variables, they must be in a separate python module +and that module must be imported with the same name in each program. (For +succinctness, I import it as "g", so that global values are referenced as +_g.variablename_.) + +The file _fssim_ is the command processor for user file system commands; it +processes commands from a command file and/or the command line. (In Unix +terms, it implements the user shell and shell commands.) + +The file _fsidxupd_ is a utility program that reads, updates, and writes +indexes. It uses file system functions from _fscommon.py_ to update a full +index using data from incremental indexes. + +## fsruntest + +The programs also include a bash shell script to automate testing. The +format of the command is: + + fsruntest [-d] + +The command file is used to run the test. It may end with an _exit_ command, +or the _exit_ command can be entered manually while the test is running. The +"-d" option runs _fssim_ and _fsidxupd_ with debugging messages enabled. + +The script first invokes _fssim_, passing in the name of the command file. It +then invokes _fsidxupd_ to apply all of the subsequent incremental indexes to +the first full index file created. Finally, it compares the output of _fsidxupd_ +to the final full index created by _fssim_. If the output of diff is not empty, +the output is displayed, preceded by a highlighted error line (so that it stands out or +can be searched for in a large test run). + +Since it is a bash shell script, the test script runs on both Linux and Mac +systems, and with the proper Linux subsystem configured perhaps even on +Windows 10 (or earlier, using Cygwin). + + +## Final Comments + +The _oids_ in the files and directories correspond to the _fileuid_ in LTFS. +Initially I did not believe that _oids_ (and thus _fileuids_) were important +to processing incremental indexes. It turns out that I was wrong, and that +a unique identifier for an object is important to the correct application +of an incremental index in one important case. Timestamps are the last +object modification time. I think all other object fields are obvious. + +The programs have not been intensively debugged. There may be lingering +errors in the commands and/or in index generation. It all seems to work, +but if you find errors (or even think you **may** have found an error) please +let me know. The better debugged the programs are, the more reliable our +results will be. + +#### - David Pease - pease@coati.com diff --git a/contrib/fssim/docs/README.txt b/contrib/fssim/docs/README.txt new file mode 100644 index 00000000..f1918670 --- /dev/null +++ b/contrib/fssim/docs/README.txt @@ -0,0 +1,393 @@ + +LTFS File System Simulator (fssim) Usage Guide and Examples (README) + +Overview +======== + +The File System Simulator programs are used to simulate file system activity, +create full and incrmental indexes reflecting that activity, update a full +index using incremental indexes, and verify the correctness of that update. + +The first program in the set is fssim. It simulates file system activity such +as creating or deleting files and directories, moving and copying files or +directories, listing information, etc. It also writes full and incremental +XML indexes to text files on request. + +The second program is fsidxupd (perhaps a poor choice of names). It updates +a full index created by fssim with one or more incremental indexes created +sequentially in the same execution of fssim, then creates a full index +from the result. The resulting full index should exactly match the +corresponding full index from fssim. + +The programs were written by David Pease at IBM's Almaden Research Center in +May-August of 2018, and significantly revised and improved by the original +author (now retired) in September of 2020. + +Invoking the programs: +====================== + +The programs are python executables, meaning that they can be invoked from +the command line in Unix-like environments (such as Linux and Mac) as long as +the executable bit is on and /usr/bin/python points to the python interpreter. +The programs are written so that they run correctly in either python 2 or +python 3. + +In Windows, it should be possible to invoke them by a command such as "python +fsXXX" or by renaming the two program files to fsXXX.py and invoking them by +name (if python is properly configured). + +There are two additional files besides the fssim and fsidxupd programs. They +are fscommon.py and fsglobals.py. Both of these are imported by the two +command programs; all four files should reside in the same directory. + +The format of the fssim command is: + + fssim [-d] [command file] + +where: + -d: prints a ton of debugging messages (probably meaningfull + only to me) during processing + command file: executes commands from the specified text file at startup + + A command file is simply a text file containing commands as they would be + entered from the keyboard; blank lines and comment lines starting with "#" + are allowed. Command files make it easy to design and debug a scenario, + then execute it multiple times (often while modifying code in-between). + After a command file is executed (without encountering an "exit" command), + the program drops into its normal keyboard input mode (which is terminated + by entering "exit"). + +The format of the fsidxupd command is: + + fsidxupd [-d] [ ...] + +where: + -d: prints lots of debugging messages during processing + fullindex: a file containing a full index from fssim + incrindexes: a list of files containing incremental indexes from fssim + +The first file name must identify a full index created by fssim. The second +and subsequent names, if present, must identify incremental indexes created +sequentially by fssim. The output of the program (currently) has a hard-coded +file name of upd-full.xml + +fssim Commands +============== + +The fssim commands are closely modeled after the Unix shell commands that +perform the same function; however, they are often greatly simplified versions +(the goal was to implement a simulator, not re-write bash). + +Unrecognized commands or commands with an invalid number of operands will +return a "** Invalid" message. Commands that do not execute successfully +(for any reason) will return the message "** Failed". That is the extent of +error messages. (This is only a test program.) + +The command prompt indicates the current working directory (as it typically +does in bash), and starts at root ("/"). File and directory names can be +entered as full paths or relative to the current directory; however, the only +place that ".." is recognized is as the single operand of a "cd" command, and +the only place that "." is recognized is as the single target for a mv or cp +command. The directory separator character is "/". The command prompt +environment supports command recall and editing (has been tested on Linux and +Mac). + +The command processor respects strings enclosed in double quotes ("). +Quoted strings can be used to create object names with embedded blanks or +to echo strings with arbitrary blank spaces. For example: + +echo Hello > "file 1" +echo " world!" >> "file 1" +cat "file 1" + +The "help" command will print a short description of all of the commands. +Here is a slightly more detailed command summary (square brackets indicate +optional parameters, curly braces indicate a choice): + +cat name - write a file's contents to the screen +cd [dirname] - change current directory; with no operands, changes to root +cls - clear the screen (tested in Linux terminal window) +cp [-r] name name - copy a file from one name to another; with -r option, + copy directory contents recursively +dir [-a] [name] - an alias for ls +echo data [{>|>>} name] - write data to the screen or to a file +fsck - verify that all object flags have been correctly reset +index [{-i|-f}] fileprefix - write full and/or incremental indexes to files + using names prefixed with the specified prefix (see below) +ll [name] - list long: an alias for ls -a +log [-t] - list current update log entries (optionally in time order) +ls [-a] [name] - list file or directory info (see more info below) +md dirname [...] - an alias for mkdir +mkdir dirname [...] - create directories or full directory paths +mv name name - move/rename a file or directory +rd dirname - an alias for rmdir +rm filename - remove a file +rmdir dirname - remove a directory (if it is empty) +touch name [...] - update the timestamp of existing files or directories, + or create new files (with directory paths if needed) +tree [-o] - show a tree view of the entire file system; with -o + option, also include oids + +The ls (dir) command prefixes each file or directory with some attributes in +the order "dmn". A "d" in the first position indicates a directory. An "m" +in the second position indicates that the file or directory has been modified +since the last index was written. An "n" in the third position indicates that +the directory is new since the last index (and thus must be written in its +entirety with all of its children to an incremental index). Following the +attributes are the object id, the modification time, and the object size. For +files, the size is the number of data bytes in the file; for directories, it +is the number of children in the directory. When the object of the "ls" +command is a directory, the "-a" (all) option will also show an entry (".") +which lists the status of the directory itself. (The ll command is shorthand +for "ls -a".) + +The tree command shows a full file system tree view; using the "-o" option +shows the object ids (oids) following the object names, as in this example: + + / (0) + |-file1 (1) + |=dir1 (2) + |=dir2 (4) + |-file2 (5) + |=dir3 (3) + +As can be seen from the example, a "-" in front of a name in the tree view +indicates a file, while a "=" indicates a directory. Within a directory +files are listed first followed by subdirectories, in alphabetical order. + +The mkdir (md), touch, and echo commands will create any missing lower-level +directories required in the directory or file path. + +A note using on the cp command with the "-r" option (recursive copy). This +command option requires two existing directory names as input; it copies the +*contents* of the first directory to the second, recursively. Thus, using a +directory structure such as that shown in the output of "tree" above, the +following command: + + cp -r dir1/dir2 dir3 + +would copy only file2 to dir3. In order to end up with dir2 and file2 under +dir3, one could use either + + cp -r dir1 dir3 + +or + md dir3/dir2 + cp -r dir1/dir2 dir3/dir2 + +Note, however, that the first option will copy everything under dir1 to dir3; +if dir1 had files or directories other than dir2 under it, it would copy +everything and would not give the desired results. + +Indexes +======= + +By default, the index command writes both a full index and an increment index +to a pair of files whose names start with the specified prefix. For example, +the command "index t0" will create files "t0-full.xml" and "t0-incr.xml" in +the user's current directory. (The rationale for writing both indices is that +that one can start from any full index written, apply any number of subsequent +incremental indexes they wish, and the result should be the same as the full +index corresponding to the last incremental index used.) The index command +allow overriding the default behavior by using the "-i" (incremental index +only) or "-f" (full index only) options. To simulate normal system operation, +(and minimize the number of files produced during testing), one could specify +"-f" for the index produced at the start of the run (e.g., "index -f t0"), then +use "-i" for all subsequent indexes up to the last one, and use neither flag +for the final index. This would give a starting and ending full index, and a +full set of incremental indexes to validate against the final full index. + +When testing indexes it's probably a good idea to start a command file with +the command "index [-f] t0" (or the prefix of your choice); this creates a +starting point for incremental indexes from the beginning of the session. +index commands can be interspersed in a command stream at any point, and the +resulting full and incremental indexes can be inspected (and processed +using fsidxupd). + +The fsck command searches the entire file system for any files or directories +with New or Modified flags. This can be used after an index command with the +"-i" (incremental only) flag to verify that all objects with these flags have +been visited, and that all such flags have been correctly reset. + +A pair of "t0" indexes (created at startup) looks like this: + + + 0 + + + + + + + 0 + + + + + +The two are essentially identical, since the root directory is the only one in +existence, and it is new and thus shows up in the incremental index as well. +(Actually, the t0 incremental index is useless, but it helps illustrate how +indexes work.) + +Invoking fssim with this set of commands (in a command file): + + index t0 + mkdir Dir1 + touch Dir2/Dir3/foo + touch Dir2/Dir3/bar + index t1 + touch Dir2/Dir3/baz + rmdir Dir1 + index t2 + +produced the following two "t2" indexes: + + + 0 + + + + Dir2 + 2 + + + + Dir3 + 3 + + + + 5 + bar + + + + 6 + baz + + + + 4 + foo + + + + + + + + + + + + + + Dir1 + + + + Dir2 + + + Dir3 + + + + baz + 6 + + + + + + + + + +Using fsidxupd +============== + +fsidxupd is used to verify the correctness of incremental indexes and their +application to a full index. For example, using the sample list of commands +in the Index section above, three sets of indexes will be created by fssim +when running the commands; the file names will be t0-full.xml, t0-incr.xml, +t1-full.xml, t1-incr.xml, t2-full.xml, and t2-incr.xml. The following +fsidxupd command will take the starting full index and apply the two appro- +priate incremental indexes: + + fsidxupd t0-full.xml t1-incr.xml t2-incr.xml + +The output of this command will be a file named upd-full.xml. In this example, +if the system is working correctly upd-full.xml will be identical to +t2-full.xml. In Unix-like systems, the command: + + diff upd-full.xml t2-full.xml + +would show no output (no differences). + +Program Organization +==================== + +The file fscommon.py contains all of the file system logic. It contains +object definitions for directory and file objects, and implements all of +the low-level logic for managing those objects. (In function, it roughly +corresponds to the vfs module in a Unix-like OS.) It also contains the +the python version-agnostic print function (fsprt). + +The file fsglobals.py defines global variables that must be shared between +the different modules. In order to provide a consistent naming reference +to these shared global variables, they must be in a separate python module +and that module must be imported with the same name in each program. (For +succinctness, I import it as "g", so that global values are referenced as +g.variablename.) + +The file fssim is the command processor for user file system commands; it +processes commands from a command file and/or the command line. (In Unix +terms, it implements the user shell and shell commands.) + +The file fsidxupd is a utility program that reads, updates, and writes +indexes. It uses file system functions from fscommon.py to update a full +index using data from incremental indexes. + +fsruntest +========= + +The programs also include a bash shell script to automate testing. The +format of the command is: + + fsruntest [-d] + +The command file is used to run the test. It may end with an "exit" command, +or the exit command can be entered manually while the test is running. The +"-d" option runs fssim and fsidxupd with debugging messages enabled. + +The script first invokes fssim, passing in the name of the command file. It +then invokes fsidxupd to apply all of the subsequent incremental indexes to +the first full index file created. Finally, it compares the output of fsidxupd +to the final full index created by fssim. If the output of diff is not empty, +the output is preceded by a highlighted error line (so that it stands out or +can be searched for in a large test run). + +Since it is a bash shell script, the test script runs on both Linux and Mac +systems, and with the proper Linux subsystem configured perhaps even on +Windows 10 (or earlier, using Cygwin). + + +Final Comments +============== + +The oids in the files and directories correspond to the fileuid in LTFS. +Initially I did not believe that oids (and thus fileuids) were important +to processing incremental indexes. It turns out that I was wrong, and that +a unique identifier for an object is important to the correct application +of an incremental index in one important case. Timestamps are the last +object modification time. I think all other object fields are obvious. + +The programs have not been intensively debugged. There may be lingering +errors in the commands and/or in index generation. It all seems to work, +but if you find errors (or even think you *may* have found an error) please +let me know. The better debugged the programs are, the more reliable our +results will be. + + -David Pease - pease@coati.com diff --git a/contrib/fssim/src/.gitignore b/contrib/fssim/src/.gitignore new file mode 100644 index 00000000..6722cd96 --- /dev/null +++ b/contrib/fssim/src/.gitignore @@ -0,0 +1 @@ +*.xml diff --git a/contrib/fssim/src/fscommon.py b/contrib/fssim/src/fscommon.py new file mode 100644 index 00000000..4df006c1 --- /dev/null +++ b/contrib/fssim/src/fscommon.py @@ -0,0 +1,713 @@ +# +# File System Simulator for LTFS Common Routines +# +# This module is imported into fssim and fsidxupd, and provides common +# values, objects, and functions used by both programs. +# +# (c) David Pease - IBM Almaden Research Center - May 2018 +# (c) David Pease - pease@coati.com - Sept. 2020 +# + +from copy import copy as pyobjcopy +from datetime import datetime +import fsglobals as g + +useNextOID = None # constant for readability +Log = {} # dictionary to hold update log entries + +########################################################################### +# object flag bit values # +########################################################################### + +modified = 0x01 # object has been modified since last index +new = 0x02 # directory is newly created since last index + +########################################################################### +# Directory object class # +########################################################################### + +class Dir(object): + def __init__(self, parent, oid=None, time=None): + self.oid = oid + self.flags = new + if time: # local code to avoid setting modified attribute + self._modTime = time + else: + self._modTime = str(datetime.now()) + self.parent = parent + self.contents = {} + @property + def oid(self): + return self._oid + @oid.setter + def oid(self, oid): + if oid is not None: + self._oid = oid + else: + self._oid = g.nextoid + g.nextoid += 1 + @property + def parent(self): + return self._parent + @parent.setter + def parent(self, parent): + self._parent = parent + @property + def modTime(self): + return self._modTime + @modTime.setter + def modTime(self, time=None): + if time: + self._modTime = time + else: + self._modTime = str(datetime.now()) + self.isModified = True + @property + def isDir(self): + return True + @property + def isFile(self): + return False + @property + def isModified(self): + return bool(self.flags & modified) + @isModified.setter + def isModified(self, ismodified): + if ismodified: + self.flags |= modified + else: + self.flags &= (0xff - modified) + @property + def isNew(self): + return bool(self.flags & new) + @isNew.setter + def isNew(self, isnew): + if isnew: + self.flags |= new + else: + self.flags &= (0xff - new) + @property + def children(self): + return self.contents.keys() + def contains(self, name): + return name in self.contents + def obj(self, name): + if not name in self.contents: + return None + return self.contents[name] + def addObj(self, name, obj, update=True): + self.contents[name] = obj + if update: + self.modTime = str(datetime.now()) + self.isModified = True + return True + def replObj(self, name, obj): + if name not in self.contents: + return False + self.contents[name] = obj + return True + def rmObj(self, name, verifyEmpty=True, update=True): + if not name in self.contents: + return False + obj = self.contents[name] + if obj.isDir and verifyEmpty: + if len(obj.contents) > 0: + return False + del self.contents[name] + if update: + self.modTime = str(datetime.now()) + self.isModified = True + return True + def __str__(self): + flags = ["new," * self.isNew, "mod," * self.isModified] + str = "Dir: oid %i, flags: '%s', parent:%i, size:%i %s %s" % \ + (self.oid, "".join(flags)[:-1], self.parent.oid, len(self.contents), + self.modTime[5:16], repr(self).split()[-1][2:-1]) + return str + +########################################################################### +# File object class # +########################################################################### + +class File(object): + def __init__(self, parent, oid=None, time=None): + self.oid = oid + self.flags = modified + self.modTime = time + self.parent = parent + self.data = "" + @property + def oid(self): + return self._oid + @oid.setter + def oid(self, oid): + if oid is not None: + self._oid = oid + else: + self._oid = g.nextoid + g.nextoid += 1 + @property + def parent(self): + return self._parent + @parent.setter + def parent(self, parent): + self._parent = parent + @property + def modTime(self): + return self._modTime + @modTime.setter + def modTime(self, time=None): + if time: + self._modTime = time + else: + self._modTime = str(datetime.now()) + self.isModified = True + @property + def isDir(self): + return False + @property + def isFile(self): + return True + @property + def isModified(self): + return bool(self.flags & modified) + @isModified.setter + def isModified(self, ismodified): + if ismodified: + self.flags |= modified + else: + self.flags &= (0xff - modified) + @property + def data(self): + return self._data + @data.setter + def data(self, data): + self._data = data if data else "" + def __str__(self): + flag = "mod" * self.isModified + str = "File: oid %i, flags:'%s', parent:%i, size:%i %s %s" % \ + (self.oid, flag, self.parent.oid, len(self.data), + self.modTime[5:16], repr(self).split()[-1][2:-1]) + return str + +########################################################################### +# python version-agnostic print function # +########################################################################### + +def fsprt(*args): + outstr = "" + for arg in args: + outstr += (str(arg) + " ") + print(outstr) + +########################################################################### +# debugging-only print function # +########################################################################### + +def dbprt(*args): + if g.debug: + fsprt(*args) + +########################################################################### +# create a log entry # +########################################################################### + +def log(name, oid, reason): + logkey = name + "#" + str(oid) + # don't overwrite a "New" log entry with a "Mod" one for the same object + if logkey in Log and Log[logkey][0] == "New": + if reason == "Mod": + dbprt("Skip log: Mod for", name, "- is already New") + return + # trim unneeded log entries if parents are New or Deleted + # (processing log entries in reverse time order - why does it matter??) + for entkey in sorted(Log, key=lambda entkey: Log[entkey][1], reverse=True): + entpath, entoid = entkey.rsplit("#", 1) + entreason, enttime = Log[entkey] + # if a parent dir is already "New", don't bother to log updates + if entreason == "New" and name.startswith(joinPath(entpath,"/")): + dbprt("Skip log", reason, "for", name, "- parent is New") + return + # if this is a parent being deleted, remove unneeded entries for children + elif reason == "DelD": + if entpath.startswith(joinPath(name, "/")): + dbprt("Remove log", entreason, "for", entpath, "- parent Deleted") + del Log[entkey] + Log[logkey] = (reason, datetime.now()) + +########################################################################### +# display log entries in pathname (processing) or time order # +########################################################################### + +def printLog(timeOrder=False): + if timeOrder: + sortkey = lambda k: Log[k][1] + else: + sortkey = None + for k in sorted(Log, key=sortkey): + if timeOrder: + fsprt(str(Log[k][1])[:-7], Log[k][0], k) + else: + fsprt("%-20s" % k, Log[k][0], str(Log[k][1])[:-7]) + return True + +########################################################################### +# file system common utility functions # +########################################################################### + +# return a fully-qualified name (with no trailing slash) +def fullName(name): + dbprt("fullName in:", name) + if name.startswith("/"): + fullname = name + else: + fullname = joinPath(g.curnm, name) + if len(fullname) > 1 and fullname.endswith("/"): + fullname = fullname[:-1] + dbprt("fullName out:", fullname) + return fullname + +# return full path and name portions of a filespec +def splitPath(name): + path,name = fullName(name).rsplit("/",1) + if not path: + path = "/" + return path,name + +# return a properly joined path and file name +def joinPath(path, name): + if name.startswith("/"): + name = name[1:] + if path.endswith("/"): + return path + name + else: + return path + "/" + name + +# process a directory path, output depends on parentRef value: +# False returns: full name of object, object instance (default behavior) +# True returns: full path to object, parent object instance, object name +def dirpath(name, parentRef=False): + fullname = fullName(name) + if not parentRef and fullname != "/": + fullname += "/" + dbprt("dirpath in:", fullname) + wdir = g.root + wdnm = "/" + parts = fullname[1:].split("/") + if parts[-1]: + lastpart = len(parts)-1 + else: + lastpart = len(parts)-2 + for cnt, dn in enumerate(parts[:-1]): + if wdir.contains(dn): + if wdir.obj(dn).isDir or cnt == lastpart: + wdir = wdir.obj(dn) + wdnm = joinPath(wdnm, dn) + continue + dbprt("dirpath out: None") + if parentRef: + return None, None, None + else: + return None, None + dbprt("dirpath out:", wdnm, wdir.oid, parts[-1]) + if parentRef: + return wdnm, wdir, parts[-1] + else: + return wdnm, wdir + +# depth-first-search of a file system tree with function invocation on descent +# parameters: +# sdobj - starting directory object (top of dfs search tree) +# sdnm - the fully-qualified name of the starting directory +# func - the function to be invoked while traversing the tree +# params - a tuple of parameters to pass to func +# return value: +# boolean - from "enter" into a directory, flag indicating whether to +# descend into (that is process the members of) that directory, +# otherwise flag from func that halts processing of dir if False +# parameters passed to func: +# reason - one of "enter", "exit", "file", or "dir" +# sdobj, sdnm, object name if "file" or "dir" else None, params +def dfs(sdobj, sdnm, func, params): + if sdnm == "/": + name = "/" + else: + name = splitPath(sdnm)[1] + descend = func("enter", sdobj, sdnm, None, params) + if descend: + for name in sorted(sdobj.children, # process files before subdirs + key=lambda name: 'x\ff'+name if sdobj.obj(name).isDir else name): + if sdobj.obj(name).isFile: + if not func("file", sdobj, sdnm, name, params): + return False + else: + if not func("dir", sdobj, sdnm, name, params): + return False + wdir = sdobj.obj(name) + wdnm = joinPath(sdnm, name) + if not dfs(wdir, wdnm, func, params): + return False + return func("leave", sdobj, sdnm, None, params) + else: + return True + +# create a new directory or file obj, add it to parent, return object instance +def makeObj(name, dir=False, oid=None, ts=None, update=True): + path, parent, newname = dirpath(name, parentRef=True) + if parent == None: + return None + if parent.contains(newname): + return None + if dir: + newobj = Dir(parent, oid=oid, time=ts) + state = "New" + else: + newobj = File(parent, oid=oid, time=ts) + state = "Mod" + log(joinPath(path, newname), newobj.oid, state) + parent.addObj(newname, newobj, update=update) + return newobj + +# remove a directory or file +def rmObj(name, dir=False, update=True): + if name == "/": + return False # can't remove root + path, parent, oldname = dirpath(name, parentRef=True) + if parent == None or not parent.contains(oldname): + return False + obj = parent.obj(oldname) + if dir: + if obj.isFile: + return False + if obj == g.curdir: + return False # can't remove current dir + else: + if obj.isDir: + return False + if not parent.rmObj(oldname, update=update): # for dir, parent will if empty + return False + deltype = "DelD" if dir else "DelF" + log(joinPath(path, oldname), obj.oid, deltype) + return True + +# print info about an object, including memory address (for debugging only) +def printObj(obj, prefix="", children=False): + fsprt(prefix+str(obj)) + if children and obj.isDir: + for child in obj.children: + fsprt(" "+str(obj.obj(child))) + +# move or rename a directory or file, or copy a single file +def movecopy(srcname, tgtname, copy=False): + # get full paths of source and target + if srcname == "/": # can't move root, copying it would be recursive + return False + if tgtname == ".": + tgtname = g.curnm + fulltgt = tgtname # save original name for later + # get source and target object and name information + srcpath, srcobj = dirpath(srcname) + tgtpath, tgtparent, tgtname = dirpath(tgtname, parentRef=True) + if not srcobj or not tgtparent: + return False + # don't allow recursive copy + if tgtpath.startswith(srcpath+"/"): + return False + # cannot copy a complete directory + if copy and srcobj.isDir: + return False + dbprt("source:", srcpath, srcobj.oid) + dbprt("target:", tgtpath, tgtparent.oid, tgtname) + srcname = splitPath(srcpath)[1] + # if target is simply "/", must have a file name to continue + if not tgtname: + tgtname = srcname + # if target is a directory, make it new parent and copy source filename + if tgtparent.contains(tgtname) and tgtparent.obj(tgtname).isDir: + tgtpath, tgtparent = dirpath(fulltgt) + tgtname = srcname + dbprt("new target info:", tgtpath, tgtname, tgtparent.oid) + # now that we have all the object info, make sure not moving object to self + srcparent = srcobj.parent + if srcparent == tgtparent and srcname == tgtname: + dbprt("Move to self!") + return False + # if target already exists, decide what needs to be done (quite complex!) + remtobj = None # flag to delete old target object + if tgtparent.contains(tgtname): + # if the target is a directory + if tgtparent.obj(tgtname).isDir: + # and the source is a file + if srcobj.isFile: + # if target contains new object name, it must be a file + if tgtparent.obj(tgtname).contains(srcname): + if tgtparent.obj(tgtname).obj(srcname).isDir: + return False + # it's a file, so we must remove it + else: + remtobj = tgtparent.obj(tgtname).obj(srcname) + # the target is a file + else: + # if the source is a directory, can't move it to a file + if srcobj.isDir: + return False + # source is a file + if not copy: + remtobj = tgtparent.obj(tgtname) + dbprt("tgt exists:", ("remove "+str(remtobj.oid)) if remtobj else "") + # we're finally ready to move something + dbprt("mvcp:", srcpath,srcname,srcobj.oid, tgtpath,tgtname,tgtparent.oid) + # if copy, create a copy of the source object + if copy: + newobj = pyobjcopy(srcobj) # new python object, copy of srcobj + newobj.oid = useNextOID + else: # move (move original object, make copy in source for deletion) + newobj = srcobj # newobj is srcobj (same python object) + srcobj = pyobjcopy(newobj) # create new python object copy for srcobj + if not srcparent.replObj(srcname, srcobj): # replace srcobj ref in parent + return False + # set target object's parent and state + newobj.parent = tgtparent + if newobj.isDir: + newobj.isNew = True + status = "New" + else: + newobj.isModified = True + status = "Mod" + # if necessary, remove existing file that is being replaced + if remtobj: + remoid = tgtparent.obj(tgtname).oid + if not tgtparent.rmObj(tgtname): + return False + log(joinPath(tgtpath, tgtname), remoid, "DelF") + # put moved/copied object in its new home + tgtparent.addObj(tgtname, newobj) # put object under target name + log(joinPath(tgtpath, tgtname), newobj.oid, status) + # for move, remove object from its old home + if not copy: + if not srcparent.rmObj(srcname, verifyEmpty=False): # rem src from parent + fsprt("Error: cannot remove source object for move; object copied.") + return False + deltype = "DelF" if srcobj.isFile else "DelD" + dbprt("Creating log:", deltype, srcpath, srcobj.oid) + log(srcpath, srcobj.oid, deltype) + if g.debug: + printObj(newobj, "mvcp new: ") + printObj(srcobj, "mvcp src: ") + return True + +# copy a directory's contents to an existing directory recursively +def copyrecurs(inloc, outloc): + # get source and target object and name information + if outloc == ".": + outloc = g.curnm + srcpath, srcobj = dirpath(inloc) + tgtpath, tgtobj = dirpath(outloc) + if not srcobj or not tgtobj: + return False + # don't allow recursive copy + if tgtpath.startswith(srcpath+"/"): + return False + # both source and target must be existing directories + if not srcobj.isDir or not tgtobj.isDir: + return False + # recursively copy files and directories + return dfs(srcobj, srcpath, copyObj, (srcpath, tgtpath)) + +# make a copy of a single file or directory (invoked through dfs()) +def copyObj(desc, dirobj, path, name, moreparams): + srcfname, tgtfname = moreparams + if desc not in ("file", "dir"): + return True + dbprt("copyObj:", desc, dirobj.oid, path, name, srcfname, tgtfname) + addpath = path[len(srcfname):] + if addpath: + tgtdir = tgtfname + addpath + else: + tgtdir = tgtfname + newname = joinPath(tgtdir, name) + if desc == "file": + oldname = joinPath(path, name) + dbprt("Copying file", oldname, "to", newname) + return movecopy(oldname, newname, copy=True) + elif desc == "dir": + dbprt("Creating directory", newname) + if not makeObj(newname, dir=True): + return False + # should copy other attributes from source to new dir here, if needed + return True + +########################################################################### +# routines used in generating indexes # +########################################################################### + +# write a full index to a file (uses dfs to descend entire tree) +def fullIndex(fn): + global indexLevel, Log + fullfn = fn + "-full.xml" + with open(fullfn, "w") as fh: + fh.write("\n") + indexLevel = 0 # used only for indentation of xml + indexEnt("dir", g.root, "/", None, ("full", fh)) + result = dfs(g.root, "/", indexEnt, ("full", fh)) + fh.write("\n") + Log = {} # clear log after writing an index + if not result: + fsprt("Error occurred creating full index") + return result + +# write an incremental index to a file +def incrIndex(fn): + global indexLevel, Log + success = True + incrfn = fn + "-incr.xml" + with open(incrfn, "w") as fh: + fh.write("\n") + indexLevel = 0 # used for indentation and to close paths + # always start with an entry for the root dir, whether or not it's in log + indexEnt("dir", g.root, "/", "", ("incr", fh)) + indexLevel += 1 + lastpath = "/" + pathindir = None + # process all log entries sorted in name,oid order + for logKey in sorted(Log): + # get full name, oid, log entry type and timestamp for this log entry + fullname, oid = logKey.rsplit("#", 1) + if fullname == "/" and oid == "0": # we already did the root dir + continue + path, name = splitPath(fullname) + reason, ts = Log[logKey] + # if debugging, show log entry + dbprt("Log entry: %s (%i) %s %s\n" % (fullname, int(oid), reason, ts)) + # get object reference for log target + obj = dirpath(fullname)[1] # already have full name, need obj + # didn't find object for this entry, error unless it was for deletion + if not obj and not reason.startswith("Del"): + dbprt("Object matching log entry not found", fullname, oid) + success = False + break + # found the same name but a different oid, also bad unless delete + if not reason.startswith("Del") and obj.oid != int(oid): + dbprt("Object oids do not match:", fullname, oid, obj.oid) + continue + # close out unneeded directories from prior object + lastdirs = lastpath.split("/")[1:] if lastpath != "/" else [] + newdirs = path.split("/")[1:] if path != "/" else [] + newLevel = len(newdirs) + lastmatch = -1 + for i in range(min(len(lastdirs), len(newdirs))+1): + if len(lastdirs) > i and len(newdirs) > i: + if lastdirs[i] == newdirs[i]: + continue + lastmatch = i-1 + break + i = 0 # inialize in case following loops null + for i in range(len(lastdirs)-1, lastmatch, -1): + fh.write(" " * (i+1) + " \n") + fh.write(" " * (i+1) + "\n") + # build new directory path structure as needed for object + for i in range(lastmatch+1, newLevel): + prefix = " " * (i+1) + temppath = "/".join(['']+newdirs[:i+1]) # full path name of dir + if temppath == pathindir: # if we just added this dir + continue # to the index, skip it + fh.write(prefix + "\n") + fh.write(prefix + " " + newdirs[i] + "\n") + tempobj = dirpath(temppath)[1] + if tempobj.isModified: # parent dir may have updated modtime + fh.write(prefix + " \n") + tempobj.isModified = False + fh.write(prefix + " \n") + indexLevel = newLevel + 1 + pathindir = None + # process Deletes here rather than using indexEnt() + if reason.startswith("Del"): + lastpath = path # make sure to save last path for deletes as well + if obj: + if obj.oid == int(oid): + dbprt("Found deleted object in file system", fullname, oid) + success = False + break + else: + continue # don't delete object if it's been recreated + prefix = " " * indexLevel + indextype = "file" if reason == "DelF" else "directory" + fh.write(prefix + "<" + indextype + ">\n") + fh.write(prefix + " " + name + "\n") + fh.write(prefix + " \n") + fh.write(prefix + "\n") + continue + # write the object entry into the index + newdir = obj.isDir and obj.isNew # indexEnt() will reset this + indextype = "file" if obj.isFile else "dir" + indexdirobj = obj.parent if obj.parent else g.root + indexEnt(indextype, indexdirobj, fullname, name, ("incr", fh)) + # if object was a New directory, output its full subtree + if obj.isDir: + if newdir: + if not dfs(obj, fullname, indexEnt, ("full", fh)): + dbprt("Error occurred in dfs for incremental index") + success = False + break + else: + pathindir = joinPath(path, name) # this feels like a kludge! + # save index state from this entry to compare to the next + lastpath = path + # close out the directory tree and the incremental index + for i in range(indexLevel-1, 0, -1): + fh.write(" " * i + " \n") + fh.write(" " * i + "\n") + fh.write(" \n") + fh.write("\n") + # clear the log and return + Log = {} + if not success: + # delete incremental index file, but for now keep it for debugging + fsprt("Error occurred creating incremental index") + return success + +# write the XML for single index entry (this function my be invoked directly, +# but is often invoked through dfs(), thus the use of params) +def indexEnt(desc, dirobj, path, name, params): + global indexLevel + enttype, fh = params + dbprt("Index entry:", desc, dirobj.oid, path, name, enttype) + if not name: # entering or leaving a directory, or root dir + obj = dirobj # obj is the directory itself + if path != "/": + name = splitPath(path)[1] + else: + obj = dirobj.obj(name) # obj is a member of the directory + if desc == "enter": + indexLevel += 1 + elif desc == "file": + prefix = " " * indexLevel + fh.write(prefix + "\n") + fh.write(prefix + " " + name + "\n") + fh.write(prefix + " " + str(obj.oid) + "\n") + # note: if enttype == "incr", should only output changed attributes + fh.write(prefix + " \n") + if obj.data or enttype == "incr": # if incr, replace any old data + fh.write(prefix + " " + obj.data + "\n") + fh.write(prefix + "\n") + obj.isModified = False + elif desc == "dir": + prefix = " " * indexLevel + if obj != g.root: + fh.write(prefix + "\n") + fh.write(prefix + " " + name + "\n") + if enttype == "full" or obj.isNew: # ??? does this match spec? + fh.write(prefix + " " + str(obj.oid) + "\n") + fh.write(prefix + " \n") + fh.write(prefix + " \n") + obj.isModified = False + obj.isNew = False + elif desc == "leave": + indexLevel -= 1 + prefix = " " * indexLevel + if enttype == "full" or obj != g.root: + fh.write(prefix + " \n") + if obj != g.root: + fh.write(prefix + "\n") + return True + diff --git a/contrib/fssim/src/fsglobals.py b/contrib/fssim/src/fsglobals.py new file mode 100644 index 00000000..12d1d84b --- /dev/null +++ b/contrib/fssim/src/fsglobals.py @@ -0,0 +1,10 @@ +# +# fssim, fsidxupd, fscommon shared global variables +# + +debug = False # debugging mode flag +nextoid = 0 # master object id +root = None # root directory object +curdir = None # current directory object +curnm = None # current directory path string + diff --git a/contrib/fssim/src/fsidxupd b/contrib/fssim/src/fsidxupd new file mode 100755 index 00000000..06991f42 --- /dev/null +++ b/contrib/fssim/src/fsidxupd @@ -0,0 +1,159 @@ +#!/usr/bin/python3 +# +# File System Index Processor for LTFS +# +# This program reads in a full index created by the fssim program, then +# updates that index using one or more incremental indexes created +# sequentially by the same program. At the end, it writes the resulting +# xml index to a file (upd-full.xml). +# +# Usage: +# +# fsidxupd [-d] [ ...] +# +# where: +# -d: prints lots of debugging messages during processing +# fullindex: a file containing a full index (from fssim) +# incrindex: a file containing an incremental index (from fssim) +# +# (c) David Pease - IBM Almaden Research Center - May 2018 +# (c) David Pease - pease@coati.com - Sept. 2020 +# + +from sys import argv, exit +from xml.etree.ElementTree import parse +from fscommon import * +import fsglobals as g + +########################################################################### +# process xml children (files and dirs) in blocks recursivley # +########################################################################### + +def processContents(path, contents, full=False): + + for child in contents: + values = {} + for val in child: + if val.tag == "contents": + values[val.tag] = val + else: + values[val.tag] = val.text + name = joinPath(path, values["name"]) + dir = child.tag == "directory" + if full: + obj = makeObj(name, oid=values["oid"], ts=values["time"], dir=dir, + update=False) + if not obj: + return False + if "data" in values: + obj.data = values["data"] + else: # is incremental + fullpath, obj = dirpath(name) + if not obj: + if "deleted" in values: + pass # already deleted, so okay + else: + dbprt("Creating:", name, values) + if not "oid" in values: + fsprt("Error creating %s, no oid in index" % name) + return False + obj = makeObj(name, oid=values["oid"], ts=values["time"], + dir=dir, update=False) + if not obj: + return False + if "data" in values: + obj.data = values["data"] + else: + if "deleted" in values: + obj.parent.rmObj(values["name"], update=False,verifyEmpty=False) + else: + if "oid" in values and obj.oid != values["oid"]: + if not obj.parent.rmObj(values["name"], update=False, + verifyEmpty=False): + return False + obj = makeObj(name, oid=values["oid"], ts=values["time"], + dir=dir, update=False) + if not obj: + return False + else: + if "time" in values: + obj.modTime = values["time"] + if "data" in values: + obj.data = values["data"] + if dir and not "deleted" in values: + dbprt("Entering:", child.tag, values) + if not processContents(name, values["contents"], full=full): + return False + return True + +########################################################################### +# main line # +########################################################################### + +# do some parameter validation +if len(argv) > 1 and argv[1] == "-d": + g.debug = True + del argv[1] +if len(argv) < 2: + fsprt("Too few parameters") + exit(12) +fullfn = argv[1] +incrlist = argv[2:] + +# first build the base filesystem from the full index +try: + xml = parse(fullfn) +except: + fsprt("Error parsing XML in file", fullfn) + exit(12) +xmlroot = xml.getroot() +if xmlroot.tag != "fullindex": + fsprt("Invalid full index file header", xmlroot.tag) + exit(12) +oid = ts = contents = None +for child in xmlroot: + if child.tag == "oid": + oid = int(child.text) + elif child.tag == "time": + ts = child.text + elif child.tag == "contents": + contents = child + else: + fsprt("Invalid tag", tag) +if oid == None or not ts or contents is None: + fsprt("Missing root directory entries") + exit(12) +g.root = Dir(None, oid=oid, time=ts) +g.curdir = g.root +g.curnm = "/" +if not processContents("/", contents, full=True): + fsprt("Error creating full index structure.") + exit(12) + +# now start applying updates from incremental indexes +for incrfn in incrlist: + try: + xml = parse(incrfn) + except: + fsprt("Error parsing XML in file", incrfn) + exit(12) + xmlroot = xml.getroot() + if xmlroot.tag != "incrementalindex": + fsprt("Invalid incremental index file header", xmlroot.tag) + exit(12) + ts = contents = None + for child in xmlroot: + if child.tag == "time": + ts = child.text + elif child.tag == "contents": + contents = child + if ts: + g.root.modTime = ts + if contents is not None: + if not processContents("/", contents): + fsprt("Error applying incremental index", incrfn) + exit(12) + +# write out final results as a full index +fullIndex("upd") + diff --git a/contrib/fssim/src/fsruntest b/contrib/fssim/src/fsruntest new file mode 100755 index 00000000..4f9e0e79 --- /dev/null +++ b/contrib/fssim/src/fsruntest @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Run an fssim test case, apply incremental indexes to initial full index, +# compare result to final full index, and display any difference +# + +# process operands and set values +if [ $# -lt 1 ]; then + echo "Usage:" $0 "[-d] " + exit 1 +fi +if [ $1 == "-d" ]; then + OPT="-d" + shift 1 +else + OPT="" +fi +if [ $# -ne 1 ]; then + echo "Missing command file name" + exit 1 +fi +if [ ! -f $1 ]; then + echo "File" $1 "does not exist" + exit 1 +fi + +# initialize xml file name suffixes +FNAME="-full.xml" +INAME="-incr.xml" + +# create list of indexes written by input command file +FLIST="" +while read -r prefix; do + FLIST="$FLIST $prefix" +done < <(grep "^ *index" $1 | rev | cut -d" " -f 1 | rev) + +# verify that at least two indexes are written by test case +alist=( $FLIST) +if [ ${#alist[@]} -lt 2 ]; then + echo "Too few indexes written in test script $1 for comparison" + exit 1 +fi + +# build file name strings based on list of indexes and name suffixes +FULLFILE="" +INCRFILES="" +for prefix in $FLIST; do + if [[ $FULLFILE == "" ]]; then + FULLFILE=$prefix$FNAME + else + INCRFILES="$INCRFILES $prefix$INAME" + COMPFILE=$prefix$FNAME + fi +done + +# build the actual command lines +CMD1="./fssim $OPT $1" +CMD2="./fsidxupd $OPT $FULLFILE $INCRFILES" +CMD3="diff upd-full.xml $COMPFILE" + +# display each command, then execute it +echo $CMD1 +$CMD1 +echo $CMD2 +$CMD2 +echo $CMD3 +$CMD3 > tempout.txt + +# check for errors and display them +if [ -s tempout.txt ]; then + echo "************** Errors from diff - $1 *****************" + cat tempout.txt +fi +rm tempout.txt diff --git a/contrib/fssim/src/fssim b/contrib/fssim/src/fssim new file mode 100755 index 00000000..be368613 --- /dev/null +++ b/contrib/fssim/src/fssim @@ -0,0 +1,376 @@ +#!/usr/bin/python3 +# +# File System Simulator for LTFS +# +# This program simulates file system directory activity such as creating or +# deleteing files and directories, moving and copying files or directories, +# listing information, etc. It also writes full and incremental XML indexes +# to files on request. +# +# Usage: +# +# fssim [-d] [command file] +# +# where: +# -d: prints lots of debugging messages during processing +# command file: executes commands from specified text file at startup +# +# (c) David Pease - IBM Almaden Research Center - May 2018 +# (c) David Pease - pease@coati.com - Sept. 2020 +# + +from sys import argv, exit, version +from fscommon import * +import fsglobals as g +import readline + +TimeNow = None # a constant value for readability + +########################################################################### +# filesystem command functions # +########################################################################### + +# cd: change directory +def cd(opt, name=None): + if name == "/" or name == None: + g.curdir = g.root + g.curnm = "/" + return True + if name == ".." and g.curnm != "/": + g.curdir = g.curdir.parent + g.curnm = splitPath(g.curnm)[0] + if g.curnm == "": + g.curnm = "/" + return True + dirnm, newdir = dirpath(name) + if newdir == None: + return False + g.curdir = newdir + g.curnm = dirnm + return True + +# md: create a directory or (multi-level) directory path +def md(opt, *names): + for name in names: + fullname = fullName(name) + pieces = fullname[1:].split("/") + path = "" + for piece in pieces: + path += ("/" + piece) + fpath, obj = dirpath(path) + if not obj: + if not makeObj(path, dir=True): + return False + return True + +# rd: remove a directory +def rd(opt, name): + return rmObj(name, dir=True) + +# rm: remove a file +def rm(opt, name): + return rmObj(name) + +# cp: copy a single file, or copy a directory's contents recursively +def cp(opt, oldname, newname): + if opt: + if opt == "-r": + return copyrecurs(oldname, newname) + else: + return False + return movecopy(oldname, newname, copy=True) + +# mv: move a file or directory +def mv(opt, oldname, newname): + return movecopy(oldname, newname) + +# tf: update an object's timestamp or create a new file +def tf(opt, *names): + for name in names: + if name == "/": # special case for touching root dir + g.root.modTime = TimeNow + log(name, 0, "Mod") + continue + path, parent, objname = dirpath(name, parentRef=True) + if parent == None: + if not md(None, splitPath(name)[0]): + return False + path, parent, objname = dirpath(name, parentRef=True) + if parent == None: + return False + if not parent.contains(objname): + if not makeObj(name): + return False + else: + parent.obj(objname).modTime = TimeNow + log(joinPath(path, objname), parent.obj(objname).oid, "Mod") + return True + +# ec: echo string to a file or the screen +def ec(opt, *args): + if len(args) >= 2 and args[-2] in (">", ">>"): + name = args[-1] + path, obj = dirpath(name) + if obj: + if obj.isDir: + return False + obj.modTime = TimeNow + else: # need to create file first + if not tf(None, name): + return False + path, obj = dirpath(name) + if not obj: + return False + data = " ".join(args[:len(args)-2]) + if args[-2] == ">>": + obj.data += data + else: + obj.data = data + log(path, obj.oid, "Mod") + else: + fsprt(*args) + return True + +# ct: cat (print) a file's data +def ct(opt, name): + path, obj = dirpath(name) + if not obj or obj.isDir: + return False + fsprt(obj.data) + return True + +# ls: list directory contents or file information with optional information +def ls(opt, dirnm=None, showall=False): + if opt: + if opt == "-a": + showall = True + else: + return False + if not dirnm: + wobj = g.curdir + dirnm = g.curnm + else: + path, wobj = dirpath(dirnm) + if not wobj: + return False + if wobj.isFile: + flag = "-m-" if wobj.isModified else "---" + lth = len(wobj.data) + fsprt(flag, wobj.oid, wobj.modTime[5:16], lth, splitPath(path)[1]) + return True + if showall: + flag = "d" +"-m"[wobj.isModified] +"-n"[wobj.isNew] + fsprt(flag, wobj.oid, wobj.modTime[5:16], len(wobj.children), ".") + for name in sorted(wobj.children, # process subdirs before files + key=lambda name: "\0"+name if wobj.obj(name).isDir else name): + if wobj.obj(name).isDir: + flag = "d" + "-m"[wobj.obj(name).isModified] + \ + "-n"[wobj.obj(name).isNew] + lth = len(wobj.obj(name).children) + else: + flag = "-" + "-m"[wobj.obj(name).isModified] + "-" + lth = len(wobj.obj(name).data) + fsprt(flag, wobj.obj(name).oid, wobj.obj(name).modTime[5:16], lth, name) + return True + +# ll: list long, same as ls -a +def ll(opt, dirnm=None): + return ls(opt, dirnm, showall=True) + +# tr: show a tree view of file system with optional oids +def tr(opt): + oids = False + if opt: + if opt == "-o": + oids = True + else: + return False + if not oids: + fsprt("/") + else: + fsprt("/ (0)") + return dfs(g.root, "/", tree, oids) + +# print out one line of the tree view (invoked through dfs()) +def tree(desc, dirobj, path, name, oids): + if desc not in ("file", "dir"): + return True + if path == "/": + prefix = "" + else: + prefix = path.count("/") * " " + if oids: + oidstr = " (" + str(dirobj.obj(name).oid) + ")" + else: + oidstr = "" + if desc == "file": + fsprt(prefix + " |-" + name + oidstr) + elif desc == "dir": + fsprt(prefix + " |=" + name + oidstr) + return True + +# cl: clear the screen +def cl(opt): + fsprt("\x1b\x5b\x48\x1b\x5b\x32\x4a") + return True + +# ix: write out both a full and an incremental index +def ix(opt, fnprefix): + full = True + incr = True + if opt: + if opt == "-f": + incr = False + elif opt == "-i": + full = False + else: + return False + if incr: + result = incrIndex(fnprefix) + if not result: + full = True # if incremental fails, force full index + if full: # full index clears log, all file flags, so must be last + result = fullIndex(fnprefix) + return result + +# pl: display update log entries by name or optionally by time +def pl(opt): + byTime = False + if opt: + if opt == "-t": + byTime = True + else: + return False + return printLog(byTime) + +# fc: check file system for extraneous or incorrect information +def fc(opt): + if g.root.isNew or g.root.isModified: + fsprt("/", ("Mod" * g.root.isModified), ("New" * g.root.isNew)) + return dfs(g.root, "/", chkMods, None) + +# display objects that indicate New or Modified (invoked thru dfs()) +def chkMods(desc, dirobj, path, name, dummy): + if desc not in ("file", "dir"): + return True + obj = dirobj.obj(name) + mod = obj.isModified + new = obj.isDir and obj.isNew + if mod or new: + fsprt(joinPath(path, name), ("Mod" * mod), ("New" * new)) + return True + +########################################################################### +# execute a single input command (from command file or keyboard) # +########################################################################### + +def execCommand(input): + if input == "exit": + return False + if input == "help": + fsprt("Enter 'exit', 'help', or one of the following commands:") + for cmdname in sorted(cmdTable): + fsprt("%-5s %s" % (cmdname, cmdTable[cmdname][3])) + return True + # tokenize input string allowing quoted strings + sstr = input.split('"') + if len(sstr) % 2 != 1: + fsprt("** Invalid") + return True + input = [] + for ct, ss in enumerate(sstr): + if not ss: + continue + if ct % 2 == 1: + input.append(ss) + continue + else: + input += ss.split() + # process the command + cmdnm = input[0] + if cmdnm not in cmdTable: + fsprt("** Invalid") + return True + func = cmdTable[cmdnm][2] + opt = None + if len(input) > 1 and input[1].startswith("-"): + opt = input[1] + del input[1] + if (len(input)-1) < cmdTable[cmdnm][0] or \ + (len(input)-1) > cmdTable[cmdnm][1]: + fsprt("** Invalid") + return True + params = [] + for p in input[1:]: + params.append(p) + if not func(opt, *params): + fsprt("** Failed") + return True + +########################################################################### +# command table - name: (min operands, max operands, function, help text) # +########################################################################### + +cmdTable = { + # cmd min max fn help text + "cd": (0, 1, cd, "[dirname] - change current directory"), + "mkdir": (1,20, md, "dirname ... - make directories"), + "md": (1,20, md, "dirname ... - alias for mkdir"), + "touch": (1,20, tf, "name ... - create file(s) or update time(s)"), + "rmdir": (1, 1, rd, "dirname - remove directory"), + "rd": (1, 1, rd, "dirname - alias for rmdir"), + "rm": (1, 1, rm, "filename - remove file"), + "cp": (2, 2, cp, "[-r] name name - copy one file, or dir recursively"), + "mv": (2, 2, mv, "name name - move/rename file or directory"), + "ls": (0, 1, ls, "[-a] [name] - list file or directory info"), + "dir": (0, 1, ls, "[-a] [name] - alias for ls"), + "ll": (0, 1, ll, "[name] - alias for ls -a (ls with all info)"), + "tree": (0, 0, tr, "[-o] - show a tree view of the file system"), + "echo": (1, 20,ec, "data [>|>> name] - write data to screen or file"), + "cat": (1, 1, ct, "name - write a file's contents to the screen"), + "cls": (0, 0, cl, " - clear the screen"), + "index": (1, 1, ix, "[-f|-i] fileprefix - write full and/or incr indexes"), + "log": (0, 0, pl, "log [-t] - display log entries by name or time"), + "fsck": (0, 0, fc, " - check files and dirs for correctness"), +} + +########################################################################### +# main line # +########################################################################### + +# create root directory with no parent and make it the current dir +g.root = Dir(None) # equivalent to mkfs -t fssim ;-) +log("/", 0, "New") +g.curdir = g.root +g.curnm = "/" + +# set debugging flag +g.debug = "-d" in argv[1:] + +# if command file specified, execute commands from it +if len(argv) > 1 and not argv[-1].startswith("-"): + infname = argv[-1] + with open(infname, "r") as infile: + inlines = infile.readlines() + for inpt in inlines: + inpt = inpt.strip() + if inpt: + if g.debug: + fsprt("==============================") + if inpt.startswith("#"): + fsprt(inpt) + else: + fsprt(g.curnm+"> "+inpt) + if not execCommand(inpt): + exit(0) + +# go into interactive mode +fsprt('Enter command, "exit", or "help":') +while True: + if version.startswith("2"): + input = raw_input # use python 3.x function name + inpt = input(g.curnm+"> ").strip() + if inpt and not inpt.startswith("#"): + if not execCommand(inpt): + exit(0) + diff --git a/contrib/fssim/testcases/daptest.txt b/contrib/fssim/testcases/daptest.txt new file mode 100644 index 00000000..75dfaee5 --- /dev/null +++ b/contrib/fssim/testcases/daptest.txt @@ -0,0 +1,11 @@ +index t0 +touch a/b/c/foo +log +index t1 +touch a/b/bar +touch a/b/baz +ll a/b +log +index t2 +exit + diff --git a/contrib/fssim/testcases/daptest2.txt b/contrib/fssim/testcases/daptest2.txt new file mode 100644 index 00000000..7f97175f --- /dev/null +++ b/contrib/fssim/testcases/daptest2.txt @@ -0,0 +1,21 @@ +index t0 +ls +ls -a +touch a/b/c/foo +touch a/b/c/bar +log +ls -a +ls -a b +ls -a a/b +md z +ls -a +ls -a a/b/c +index t1 +ll +ll a +md c +ll +ls +rd c +index t2 +exit diff --git a/contrib/fssim/testcases/daptest3.txt b/contrib/fssim/testcases/daptest3.txt new file mode 100644 index 00000000..58144bd5 --- /dev/null +++ b/contrib/fssim/testcases/daptest3.txt @@ -0,0 +1,10 @@ +index t0 +touch foo +echo "Hi" > bar +ll +log +mv bar foo +ll +log +index t1 +exit diff --git a/contrib/fssim/testcases/daptest4.txt b/contrib/fssim/testcases/daptest4.txt new file mode 100644 index 00000000..22cfad72 --- /dev/null +++ b/contrib/fssim/testcases/daptest4.txt @@ -0,0 +1,7 @@ +index t0 +touch a/b/foo +touch a/c/foo +index t1 +touch a/c/foo +index t2 +exit diff --git a/contrib/fssim/testcases/daptest5.txt b/contrib/fssim/testcases/daptest5.txt new file mode 100644 index 00000000..130a219f --- /dev/null +++ b/contrib/fssim/testcases/daptest5.txt @@ -0,0 +1,34 @@ +index t30 +# /A +# |_ /B +# | |_ n1 +# |_ n2 +# |_ n3 +# /C +# |_ n4 +# |_ n5 +# /n1 +# /D +# /E +# +touch /A/B/n1 /A/n2 /A/n3 /C/n4 /C/n5 n1 +mkdir D E +tree +index t31 +# 1. Delete /A/B/n1 +# Rename the directory /C to /A/B/n1 +rm /A/B/n1 +mv /C /A/B/n1 +tree -o +index t32 +# 2. Append data data to /A/B/n1 +# Delete /A/B/n1 +# Rename file /n1 to /A/B/n1 +touch /A/B/n1 +rm /A/B/n1 +mv n1 /A/B/n1 +tree -o +log +index t33 +exit + diff --git a/contrib/fssim/testcases/daptest6.txt b/contrib/fssim/testcases/daptest6.txt new file mode 100644 index 00000000..bfb2f56f --- /dev/null +++ b/contrib/fssim/testcases/daptest6.txt @@ -0,0 +1,9 @@ +index q0 +touch a/b/c/foo +index q1 +touch a/b/c/foo +touch a/b/c +touch a/b +log +index q2 +exit diff --git a/contrib/fssim/testcases/fssimin-Log.txt b/contrib/fssim/testcases/fssimin-Log.txt new file mode 100644 index 00000000..9618232b --- /dev/null +++ b/contrib/fssim/testcases/fssimin-Log.txt @@ -0,0 +1,14 @@ +index l0 +touch A/D/bar +touch A/B/C/foo +echo "Hello world!" > A/B/C/foo +cd /A/B/C +cp foo / +cd +rm A/B/C/foo +rm foo +echo "foo!" > foo +log +log -t +index l1 +exit diff --git a/contrib/fssim/testcases/fssimin-TI.txt b/contrib/fssim/testcases/fssimin-TI.txt new file mode 100644 index 00000000..9998490f --- /dev/null +++ b/contrib/fssim/testcases/fssimin-TI.txt @@ -0,0 +1,77 @@ +index J0 +touch A1/B1/C1/c +echo "some " > A1/B1/b +md A1/B2 A1/B3 +tree +index J1 +#Case 1. Deleting a file +# step 1: delete /A1/B1/b +rm A1/B1/b +tree +index J2 +#Case 2. Adding data to a file, and rename the file +# step 1: adding data at end of existing file /A1/B1/b +# step 2: rename the file b to b.NEW +echo data >> A1/B1/b +mv A1/B1/b A1/B1/b.NEW +tree +index J3 +#Case 3: 1. Deleting a file (file UID: XXX), +# and create a new file with same name (file UID: YYY) +touch foo +ls +rm foo +touch foo +ls +index J4 +rm foo +#Case 4. Creating a new file with same name, after renaming the original one +# step 1: rename file b to b.bak +# step 2: create a new file as b +touch b +mv b b.bak +touch b +ls +index J5 +#Case 5. Moving whole subdirectory to another location +# step 1: move directory B1 under directory B2 +tree +mv A1/B1 A1/B2/B1 +tree +index J6 +#Case 6. Moving a file to other location, following by moving its parent +# directory to different location (or move a subdirectory first, and then +# move a file inside the subdirectory to other location). +# step 1: move file /A1/B1/C1/c to B2 (now it becomes /A1/B2/c) +# step 2: move directory B1 (not including c) under B3 +# or, +# step 1: move directory B1 under B3 +# step 2: move file c in /A1/B3/B1/C1 directory to B2 +md A1/B1/C1 A1/B3 +touch A1/B1/C1/c +tree +index XY +mv A1/B1/C1/c A1/B2 +mv A1/B1 A1/B3 +tree -o +log +index J7 +#Case 7. Moving a file more than once +# step 1: move file b to directory B2 +# step 2: move the same file (/A1/B2/b) to B3 +mv b /A1/B2 +ll +ls /A1/B2 +mv /A1/B2/b A1/B3 +ll /A1/B2 +ls /A1/B3 +tree +index J8 +#Case 8. Create a file and delete it within an interval +# step 1: delete file c +# step 2: create file c in /A1/B1/C +touch c +rm c +ll +index J9 +exit diff --git a/contrib/fssim/testcases/fssimin10.txt b/contrib/fssim/testcases/fssimin10.txt new file mode 100644 index 00000000..13902adc --- /dev/null +++ b/contrib/fssim/testcases/fssimin10.txt @@ -0,0 +1,63 @@ +index -f t10-0 +# /A +# |_ /B +# | |_ n1 +# |_ n2 +# |_ n3 +# /C +# |_ n4 +# |_ n5 +# /n1 +# /D +# /E +# +touch /A/B/n1 /A/n2 /A/n3 /C/n4 /C/n5 n1 +mkdir D E +tree +index -i t10-1 +fsck +# 1. Delete /A/B/n1 +# Rename the directory /C to /A/B/n1 +rm /A/B/n1 +mv /C /A/B/n1 +tree -o +index -i t10-2 +fsck +# 2. Append data data to /A/B/n1 +# Delete /A/B/n1 +# Rename file /n1 to /A/B/n1 +touch /A/B/n1 +rm /A/B/n1 +mv n1 /A/B/n1 +tree -o +index -i t10-3 +fsck +# 3. Delete /E +# Rename /D to /E +rd E +mv D E +tree +index -i t10-4 +fsck +# 4. Append data to /A/B/n1 +# Rename /A/B/n1 to /A/n1 +# Delete /A/B +# Rename /C to /A/B +# Move back /A/n1 to the new directory under /A/B +md C +touch /A/B/n1 +mv A/B/n1 A/n1 +rd A/B +mv C /A/B +mv n1 /A/B +tree +log +index t10-5 +fsck +# 5. Delete /n1 +# Create a new non-empty file /n1 +# Delete /n1 again +# Create an empty file /n1 + +exit + diff --git a/contrib/fssim/testcases/fssimin4.txt b/contrib/fssim/testcases/fssimin4.txt new file mode 100644 index 00000000..aa5cc5f5 --- /dev/null +++ b/contrib/fssim/testcases/fssimin4.txt @@ -0,0 +1,7 @@ +index t4-0 +touch asdf/foo +rm asdf/foo +rd asdf +index t4-1 +exit + diff --git a/contrib/fssim/testcases/fssimin5.txt b/contrib/fssim/testcases/fssimin5.txt new file mode 100644 index 00000000..036efda7 --- /dev/null +++ b/contrib/fssim/testcases/fssimin5.txt @@ -0,0 +1,8 @@ +index t5-0 +touch asdf/qwer/zxcv/foo +index t5-1 +rm asdf/qwer/zxcv/foo +ll asdf/qwer/zxcv +index t5-2 +ll asdf/qwer/zxcv +exit diff --git a/contrib/fssim/testcases/fssimin6.txt b/contrib/fssim/testcases/fssimin6.txt new file mode 100644 index 00000000..761dc985 --- /dev/null +++ b/contrib/fssim/testcases/fssimin6.txt @@ -0,0 +1,8 @@ +index t6-0 +md Dir1 +touch Dir2/Dir3/foo Dir2/Dir3/bar +index t6-1 +touch Dir2/Dir3/baz +rd Dir1 +index t6-2 +exit diff --git a/contrib/fssim/testcases/fssimin7.txt b/contrib/fssim/testcases/fssimin7.txt new file mode 100644 index 00000000..0638fba9 --- /dev/null +++ b/contrib/fssim/testcases/fssimin7.txt @@ -0,0 +1,35 @@ +index t7-0 +touch asdf/foo asdf/bar +rm asdf/foo +rm asdf/bar +# asdf has deleted files foo and bar +ll asdf +rd asdf +ll +# asdf is deleted +md asdf +ll +# asdf is recreated, has a different oid (so is different directory) +ll asdf +rd asdf +# asdf is deleted again +ll +touch qwer/baz qwer/bar +# qwer is created, contains only bar and baz +ll +ll qwer +tree -o +mv qwer asdf +# qwer (with bar and baz) is moved to deleted asdf, asdf has new oid +ll +# qwer is now deleted, asdf is recreated (share same oid) +ll asdf +# asdf should contain bar, baz +md qwer +# qwer is recreated, gets new oid so is different directory +ll +ll asdf +ll qwer +tree -o +index t7-1 +exit diff --git a/contrib/fssim/testcases/fssimin8.txt b/contrib/fssim/testcases/fssimin8.txt new file mode 100644 index 00000000..f438efdb --- /dev/null +++ b/contrib/fssim/testcases/fssimin8.txt @@ -0,0 +1,18 @@ +index t8-0 +touch asdf/foo asdf/bar +index t8-1 +# index t8-1 contains asdf with both foo and bar +rm asdf/foo +rm asdf/bar +rd asdf +# remove foo and bar, and directory asdf +touch asdf +#recreate asdf as a file +rm asdf +# now remove asdf +md asdf +# and recreate asdf as a directory again +index t8-2 +tree +exit + diff --git a/contrib/fssim/testcases/fssimin9.txt b/contrib/fssim/testcases/fssimin9.txt new file mode 100644 index 00000000..a55f2638 --- /dev/null +++ b/contrib/fssim/testcases/fssimin9.txt @@ -0,0 +1,17 @@ +index t9-0 +md qwer +touch asdf/zxcv/uiop/foo asdf/zxcv/uiop/bar asdf/zxcv/hjkl/baz +tree -o +# directory is qwer empty, adsf contains zxcv with subdirs and files +index t9-1 +mv asdf/zxcv qwer +tree -o +ll asdf +# move entire zxcv subtree to qwer, verify everything moves correctly +index t9-2 +md asdf/zxcv +cp -r qwer/zxcv asdf/zxcv +tree -o +index t9-3 +exit + diff --git a/contrib/fssim/testcases/logerr.txt b/contrib/fssim/testcases/logerr.txt new file mode 100644 index 00000000..df776b5d --- /dev/null +++ b/contrib/fssim/testcases/logerr.txt @@ -0,0 +1,19 @@ +touch A/B/foo +touch A/B/bar +touch A/B/baz +touch A/B/C/foo +touch A/B/C/bar +touch A/B/C/baz +log +index l0 +rm A/B/foo +rm A/B/bar +rm A/B/baz +rm A/B/C/foo +rm A/B/C/bar +rm A/B/C/baz +rd A/B/C +rd A/B +log +index l1 +exit diff --git a/contrib/fssim/testcases/tc001.txt b/contrib/fssim/testcases/tc001.txt new file mode 100644 index 00000000..2360faca --- /dev/null +++ b/contrib/fssim/testcases/tc001.txt @@ -0,0 +1,14 @@ +mkdir /DA1 +mkdir /DB1/DB2/DB3/DB4/DB5/DB6/DB7/DB8/DB9/DB10 +index initial_tc001 +mv /DB1/DB2/DB3/DB4/DB5/DB6/DB7/DB8/DB9/DB10 /DA1 +mv /DB1/DB2/DB3/DB4/DB5/DB6/DB7/DB8/DB9 /DA1/DB10 +mv /DB1/DB2/DB3/DB4/DB5/DB6/DB7/DB8 /DA1/DB10/DB9 +mv /DB1/DB2/DB3/DB4/DB5/DB6/DB7 /DA1/DB10/DB9/DB8 +mv /DB1/DB2/DB3/DB4/DB5/DB6 /DA1/DB10/DB9/DB8/DB7 +mv /DB1/DB2/DB3/DB4/DB5 /DA1/DB10/DB9/DB8/DB7/DB6 +mv /DB1/DB2/DB3/DB4 /DA1/DB10/DB9/DB8/DB7/DB6/DB5 +mv /DB1/DB2/DB3 /DA1/DB10/DB9/DB8/DB7/DB6/DB5/DB4 +mv /DB1/DB2 /DA1/DB10/DB9/DB8/DB7/DB6/DB5/DB4/DB3 +index flipped_tc001 +exit diff --git a/contrib/fssim/testcases/tc002.txt b/contrib/fssim/testcases/tc002.txt new file mode 100644 index 00000000..a901c792 --- /dev/null +++ b/contrib/fssim/testcases/tc002.txt @@ -0,0 +1,8 @@ +mkdir /DB0 +mkdir /DB1/DB2/DB3/DB4/DB5/DB6/DB7/DB8/DB9/DB10 +index initial_tc002 +mv /DB1/DB2 /DB0 +mv /DB0/DB2/DB3 /DB1 +mv /DB1/DB3/DB4 /DB0/DB2 +index shuffle_tc002 +exit diff --git a/contrib/fssim/testcases/tc003.txt b/contrib/fssim/testcases/tc003.txt new file mode 100644 index 00000000..e83016ee --- /dev/null +++ b/contrib/fssim/testcases/tc003.txt @@ -0,0 +1,29 @@ +mkdir /DB0 +mkdir /DB1/DB2/DB3/DB4/DB5/DB6/DB7/DB8/DB9/DB10 +tree +index initial_tc003 +mv /DB1/DB2 /DB0 +mv /DB0/DB2/DB3 /DB1 +mv /DB1/DB3/DB4 /DB0/DB2 +mv /DB0/DB2/DB4/DB5 /DB1/DB3 +mv /DB1/DB3/DB5/DB6 /DB0/DB2/DB4 +mv /DB0/DB2/DB4/DB6/DB7 /DB1/DB3/DB5 +mv /DB1/DB3/DB5/DB7/DB8 /DB0/DB2/DB4/DB6 +mv /DB0/DB2/DB4/DB6/DB8/DB9 /DB1/DB3/DB5/DB7 +mv /DB1/DB3/DB5/DB7/DB9/DB10 /DB0/DB2/DB4/DB6/DB8 +tree -o +mv /DB0 /DA1 +mv /DA1/DB2 /DA1/DA2 +mv /DA1/DA2/DB4 /DA1/DA2/DA3 +mv /DA1/DA2/DA3/DB6 /DA1/DA2/DA3/DA4 +mv /DA1/DA2/DA3/DA4/DB8 /DA1/DA2/DA3/DA4/DA5 +mv /DA1/DA2/DA3/DA4/DA5/DB10 /DA1/DA2/DA3/DA4/DA5/DA6 +mv /DB1/DB3/DB5/DB7/DB9 /DB1/DB3/DB5/DB7/DB1 +mv /DB1/DB3/DB5/DB7 /DB1/DB3/DB5/DB2 +mv /DB1/DB3/DB5 /DB1/DB3/DB3 +mv /DB1/DB3 /DB1/DB4 +mv /DB1 /DB5 +tree -o +#log +index shuffle_tc003 +exit diff --git a/contrib/fssim/testcases/tc004.txt b/contrib/fssim/testcases/tc004.txt new file mode 100644 index 00000000..de81eaab --- /dev/null +++ b/contrib/fssim/testcases/tc004.txt @@ -0,0 +1,34 @@ +mkdir /DB0 +mkdir /DB1/DB2/DB3/DB4/DB5/DB6/DB7/DB8/DB9/DB10 +index initial_tc004 +mv /DB1/DB2 /DB0 +mv /DB0/DB2/DB3 /DB1 +mv /DB1/DB3/DB4 /DB0/DB2 +mv /DB0/DB2/DB4/DB5 /DB1/DB3 +mv /DB1/DB3/DB5/DB6 /DB0/DB2/DB4 +mv /DB0/DB2/DB4/DB6/DB7 /DB1/DB3/DB5 +mv /DB1/DB3/DB5/DB7/DB8 /DB0/DB2/DB4/DB6 +mv /DB0/DB2/DB4/DB6/DB8/DB9 /DB1/DB3/DB5/DB7 +mv /DB1/DB3/DB5/DB7/DB9/DB10 /DB0/DB2/DB4/DB6/DB8 +mv /DB0 /DA1 +mv /DA1/DB2 /DA1/DA2 +mv /DA1/DA2/DB4 /DA1/DA2/DA3 +mv /DA1/DA2/DA3/DB6 /DA1/DA2/DA3/DA4 +mv /DA1/DA2/DA3/DA4/DB8 /DA1/DA2/DA3/DA4/DA5 +mv /DA1/DA2/DA3/DA4/DA5/DB10 /DA1/DA2/DA3/DA4/DA5/DA6 +mv /DB1/DB3/DB5/DB7/DB9 /DB1/DB3/DB5/DB7/DB1 +mv /DB1/DB3/DB5/DB7 /DB1/DB3/DB5/DB2 +mv /DB1/DB3/DB5 /DB1/DB3/DB3 +mv /DB1/DB3 /DB1/DB4 +mv /DB1 /DB5 +tree +rmdir /DA1/DA2/DA3/DA4/DA5/DA6 +rmdir /DA1/DA2/DA3/DA4/DA5 +rmdir /DA1/DA2/DA3/DA4 +rmdir /DA1/DA2/DA3 +rmdir /DB5/DB4/DB3/DB2/DB1 +rmdir /DB5/DB4/DB3/DB2 +rmdir /DB5/DB4/DB3 +tree +index shuffle_remove_tc004 +exit diff --git a/contrib/fssim/testcases/tc005.txt b/contrib/fssim/testcases/tc005.txt new file mode 100644 index 00000000..04620dad --- /dev/null +++ b/contrib/fssim/testcases/tc005.txt @@ -0,0 +1,13 @@ +mkdir /DA0 +mkdir /DB0 +touch /DA0/fa0 +touch /DB0/fb0 +tree +index initial_tc005 +rm /DA0/fa0 +touch /DA0/fa0 +mv /DB0/fb0 /DB0/fb1 +mv /DB0/fb1 /DB0/fb0 +tree +index new_samename_tc005 +exit diff --git a/contrib/ut-incindex/README.md b/contrib/ut-incindex/README.md new file mode 100644 index 00000000..ec3d9658 --- /dev/null +++ b/contrib/ut-incindex/README.md @@ -0,0 +1,13 @@ +# Unit Tests for incremental index + +This is a directory for having unit tests for incremental index feature. + +## How to run + +### Basic operation test + + 1. `cd` to this directory + 2. Run the basic test with `./ut-basic.sh [mount_point]` + - The test script formats a (filebackend) tape under `/tmp/ltfstape`, start ltfs and stop ltfs automatically. + - If `[mount_point]` is not specified, the script uses `/tmp/mnt` + - You can pass the specific `ltfs` binary directory with teh environmental value `LTFS_BIN_PATH` diff --git a/contrib/ut-incindex/ut-basic.sh b/contrib/ut-incindex/ut-basic.sh new file mode 100755 index 00000000..de61a29b --- /dev/null +++ b/contrib/ut-incindex/ut-basic.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +source ./utils.sh + +MOUNTPOINT='/tmp/mnt' +TAPE_PATH='/tmp/ltfstape' + +if [ $# == 1 ]; then + MOUNTPOINT=$1 +elif [ $# -gt 2 ]; then + MOUNTPOINT=$1 + TAPE_PATH=$2 +fi + +# Format LTFS +FormatLTFS ${TAPE_PATH} +if [ $? != 0 ]; then + exit 1 +fi + +# Launch LTFS +LaunchLTFS ${MOUNTPOINT} ${TAPE_PATH} +if [ $? != 0 ]; then + exit 1 +fi + +# 1. CREATE DIRS +# Create a few dirs and files but all objects are handles by new dirs +echo "1. CREATE DIRS" +mkdir -p ${MOUNTPOINT}/dir1/dir11 +mkdir -p ${MOUNTPOINT}/dir1/dir12 +mkdir -p ${MOUNTPOINT}/dir2/dir21 +mkdir -p ${MOUNTPOINT}/dir2/dir22 +echo "AAA" > ${MOUNTPOINT}/dir1/file11 +echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file111 +echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file112 +IncrementalSync ${MOUNTPOINT} '1.CREATE_DIRS' +if [ $? != 0 ]; then + exit 1 +fi + +# 2. CREATE FILES +# Create files for checking file creation and directory traverse +echo "2. CREATE FILES" +echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file113 +echo "AAA" > ${MOUNTPOINT}/dir1/dir12/file121 +echo "AAA" > ${MOUNTPOINT}/dir2/dir22/file221 +echo "AAA" > ${MOUNTPOINT}/dir2/file21 +IncrementalSync ${MOUNTPOINT} '2.CREATE_FILES' +if [ $? != 0 ]; then + exit 1 +fi + +# 3. MODIFY FILES +# Modify contents of files. Need to check /dir2 doesn't have meta-data on the incremental index +echo "3. MODIFY FILES" +echo "BBB" > ${MOUNTPOINT}/dir1/dir11/file111 +echo "BBB" > ${MOUNTPOINT}/dir2/dir22/file221 +echo "BBB" > ${MOUNTPOINT}/dir1/file11 +IncrementalSync ${MOUNTPOINT} '3.MODIFY_FILES' +if [ $? != 0 ]; then + exit 1 +fi + +# 4. MODIFY DIRS +# Modify directory's meta-data. Need to check both /dir1 and /dir1/dir11 has meta-data +# on the incremental index +echo "4. MODIFY DIRS" +AddXattr ${MOUNTPOINT}/dir1 'ut-attr1' 'val1' +echo "CCC" > ${MOUNTPOINT}/dir1/dir11/file111 +IncrementalSync ${MOUNTPOINT} '4.MODIFY_DIRS' +if [ $? != 0 ]; then + exit 1 +fi + +# 5. DELETE FILES +echo "5. DELETE FILES" +rm ${MOUNTPOINT}/dir1/dir11/* +IncrementalSync ${MOUNTPOINT} '5.DELETE_FILES' +if [ $? != 0 ]; then + exit 1 +fi + +# 6. DELETE DIR +echo "5. DELETE DIR" +rm -rf ${MOUNTPOINT}/dir1/dir11 +IncrementalSync ${MOUNTPOINT} '6.DELETE_DIR' +if [ $? != 0 ]; then + exit 1 +fi + +# Stop LTFS +StopLTFS ${MOUNTPOINT} ${TAPE_PATH} +if [ $? != 0 ]; then + exit 1 +fi diff --git a/contrib/ut-incindex/utils.sh b/contrib/ut-incindex/utils.sh new file mode 100755 index 00000000..ca79d3c6 --- /dev/null +++ b/contrib/ut-incindex/utils.sh @@ -0,0 +1,197 @@ +#!/bin/sh + +PLATFORM=`uname` +ECHO='/bin/echo' + +if [ "x${LTFS_BIN_PATH}" == 'x' ]; then + LTFS_BIN_PATH='/usr/local/bin' +fi + +AddXattr() +{ + if [ "x$1" == "x" ]; then + "Need to a target to set xattr" + return 1 + else + TARGET=$1 + fi + + if [ "x$2" == "x" ]; then + "Need to a name to set xattr" + return 1 + else + NAME=$2 + fi + + if [ "x$3" == "x" ]; then + "Need to a value to set xattr" + return 1 + else + VAL=$2 + fi + + ${ECHO} -n "Setting a xattr ${NAME} to ${TARGET} ... " + if [ "$PLATFORM" == "Darwin" ]; then + /usr/bin/xattr -w ${NAME} ${VAL} ${TARGET} + else + /usr/bin/attr -s ${NAME} -V ${VAL} ${TARGET} + fi + + if [ $? == 0 ]; then + ${ECHO} "Done" + return 0 + else + ${ECHO} "Failed" + return 1 + fi + +} + +IncrementalSync() +{ + if [ "x$1" == "x" ]; then + MOUNTPOINT='/tmp/mnt' + else + MOUNTPOINT=$1 + fi + + if [ "x$2" == "x" ]; then + MSG='Incremental Sync' + else + MSG=$2 + fi + + ${ECHO} -n "Syncing LTFS (Incremental) ... " + if [ "$PLATFORM" == "Darwin" ]; then + /usr/bin/xattr -w ltfs.vendor.IBM.IncrementalSync ${MSG} ${MOUNTPOINT} + else + /usr/bin/attr -s ltfs.vendor.IBM.IncrementalSync -V ${MSG} ${MOUNTPOINT} + fi + + if [ $? == 0 ]; then + ${ECHO} "Done" + return 0 + else + ${ECHO} "Failed" + return 1 + fi +} + +FullSync() +{ + if [ "x$1" == "x" ]; then + MOUNTPOINT='/tmp/mnt' + else + MOUNTPOINT=$1 + fi + + if [ "x$2" == "x" ]; then + MSG='Full Sync' + else + MSG=$2 + fi + + ${ECHO} "Syncing LTFS (Full) ... " + if [ "$PLATFORM" == "Darwin" ]; then + /usr/bin/xattr -w ltfs.vendor.IBM.FullSync ${MSG} ${MOUNTPOINT} + else + /usr/bin/attr -s ltfs.vendor.IBM.FullSync -V ${MSG} ${MOUNTPOINT} + fi + + if [ $? == 0 ]; then + ${ECHO} "Done" + return 0 + else + ${ECHO} "Failed" + return 1 + fi +} + +FormatLTFS() +{ + if [ "x$1" == "x" ]; then + TAPE_PATH='/tmp/ltfstape' + else + TAPE_PATH=$1 + fi + + if [ ! -d ${TAPE_PATH} ]; then + ${ECHO} "Creating tape directory for file backend: ${TAPE_PATH}" + mkdir -p ${TAPE_PATH} + if [ $? != 0 ]; then + ${ECHO} "Failed to create a tape path: ${TAPE_PATH}" + return 1 + fi + fi + + ${ECHO} "Formatting tape directory with the file backend on ${TAPE_PATH} ... " + ${LTFS_BIN_PATH}/mkltfs -f -e file -d ${TAPE_PATH} + if [ $? != 0 ]; then + ${ECHO} "Failed to format a tape path: ${TAPE_PATH}" + return 1 + fi + + ${ECHO} "Formatted the file backend on ${TAPE_PATH}" + return 0 +} + +LaunchLTFS() +{ + if [ "x$1" == "x" ]; then + MOUNTPOINT='/tmp/mnt' + else + MOUNTPOINT=$1 + fi + + if [ "x$2" == "x" ]; then + TAPE_PATH='/tmp/ltfstape' + else + TAPE_PATH=$2 + fi + + if [ ! -d ${MOUNTPOINT} ]; then + ${ECHO} "Creating mount point for LTFS: ${MOUNTPOINT}" + mkdir -p ${MOUNTPOINT} + if [ $? != 0 ]; then + ${ECHO} "Failed to create a mount point" + return 1 + fi + fi + + if [ ! -d ${TAPE_PATH} ]; then + ${ECHO} "Creating tape directory for file backend: ${TAPE_PATH}" + mkdir -p ${TAPE_PATH} + if [ $? != 0 ]; then + ${ECHO} "Failed to create a tape path: ${TAPE_PATH}" + return 1 + fi + + ${ECHO} "Formatting tape directory with the file backend" + ${LTFS_BIN_PATH}/mkltfs -f -e file -d ${TAPE_PATH} + if [ $? != 0 ]; then + ${ECHO} "Failed to format a tape path: ${TAPE_PATH}" + return 1 + fi + fi + + ${ECHO} "Launching LTFS with the file backend" + ${LTFS_BIN_PATH}/ltfs -o tape_backend=file -o sync_type=unmount -o devname=${TAPE_PATH} ${MOUNTPOINT} + if [ $? != 0 ]; then + ${ECHO} "Failed to launch LTFS on ${MOUNTPOINT}" + return 1 + fi + + ${ECHO} "LTFS is launched on ${MOUNTPOINT}" + return 0 +} + +StopLTFS() +{ + if [ "x$1" == "x" ]; then + MOUNTPOINT='/tmp/mnt' + else + MOUNTPOINT=$1 + fi + + sudo umount ${MOUNTPOINT} +} diff --git a/man/Makefile.am b/man/Makefile.am index f18e081d..f6eec413 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -1,4 +1,4 @@ -man_MANS = mkltfs.8 ltfsck.8 ltfs-sde.8 ltfs_ordered_copy.1 +man_MANS = mkltfs.8 ltfsck.8 ltfs.8 ltfs_ordered_copy.1 ltfsindextool.8 man-clean: rm $(man_MANS) @@ -6,6 +6,7 @@ man-clean: man-rebuild: docbook2man sgml/mkltfs.sgml docbook2man sgml/ltfsck.sgml - docbook2man sgml/ltfs-sde.sgml + docbook2man sgml/ltfs.sgml docbook2man sgml/ltfs_ordered_copy.sgml + docbook2man sgml/ltfsindextool.sgml rm -f manpage.* diff --git a/man/ltfs-sde.8 b/man/ltfs.8 similarity index 87% rename from man/ltfs-sde.8 rename to man/ltfs.8 index 3ac31bf9..e159333e 100644 --- a/man/ltfs-sde.8 +++ b/man/ltfs.8 @@ -1,5 +1,5 @@ .\" auto-generated by docbook2man-spec from docbook-utils package -.TH "LTFS-SDE" "8" "16 July 2020" "IBM Spectrum Archive" "IBM Spectrum Archive Command Reference" +.TH "LTFS" "8" "21 February 2022" "LTFS" "LTFS Command Reference" .SH NAME ltfs \- File system based on a linear tape drive .SH SYNOPSIS @@ -70,8 +70,11 @@ Only use for a CM corrupted medium \fB-o device_list\fR Show available tape devices .TP -\fB-o rollback_mount=\fIgen\fB\fR -Attempt to mount on previous index generation (read-only mount) +\fB-o rollback_mount=\fIgen|index_file\fB\fR +Attempt to mount on previous index generation on tape or specified index file by read-only mode + +When both index_file and device name is specified on -o devname option, normal read-only mount is attempted. +When index_file is only specified, meta-data read-only mode is attempted. EAGAIN would be returned at accessing contents of file. .TP \fB-o release_device\fR Clear device reservation (should be specified with -o devname @@ -179,8 +182,10 @@ Override output verbosity directly (default: 2) \fB-o noeject\fR Do not eject the cartridge after unmount (default) .TP -\fB-o capture_index\fR -Capture latest index to work directory at unmount +\fB-o capture_index=\fIdir\fB\fR +Capture index to the specified directory by dir when index is updated. +File name of each index is [BARCODE]-[GEN]-[PARTITION].xml if tape serial (barcode) +is specified at format time. Otherwise it is [VOL_UUID]-[GEN]-[PARTITION].xml. .TP \fB-o scsi_append_only_mode=\fIon|off\fB\fR Set the tape device append-only mode (default=on) diff --git a/man/ltfs_ordered_copy.1 b/man/ltfs_ordered_copy.1 index 0420c469..21ccc447 100644 --- a/man/ltfs_ordered_copy.1 +++ b/man/ltfs_ordered_copy.1 @@ -1,10 +1,10 @@ .\" auto-generated by docbook2man-spec from docbook-utils package -.TH "LTFS_ORDERED_COPY" "1" "16 July 2020" "IBM Spectrum Archive" "IBM Spectrum Archive Command Reference" +.TH "LTFS_ORDERED_COPY" "1" "19 August 2021" "LTFS" "LTFS Command Reference" .SH NAME ltfs_ordered_copy \- Copy files from source to destination with LTFS order optimization .SH SYNOPSIS .sp -\fBltfs_ordered_copy\fR [ \fB-p\fR ] [ \fB-r\fR ] [ \fB-t \fITARGET_DIRECTORY\fB\fR ] [ \fB--keep-tree \fICUTOFF_PREFIX\fB\fR ] [ \fB-a\fR ] [ \fB-v\fR ] [ \fB--verbose \fILOG_LEVEL\fB\fR ] [ \fB-q\fR ] [ \fB-h\fR ] [ \fBSOURCE \fR\fI...\fR ] [ \fBDESTINATION\fR ] +\fBltfs_ordered_copy\fR [ \fB-p\fR ] [ \fB-r\fR ] [ \fB-t \fITARGET_DIRECTORY\fB\fR ] [ \fB--keep-tree \fICUTOFF_PREFIX\fB\fR ] [ \fB-a\fR ] [ \fB-v\fR ] [ \fB--verbose \fILOG_LEVEL\fB\fR ] [ \fB-q\fR ] [ \fB-h\fR ] [ \fBSOURCE \fR\fI...\fR ] [ \fBDESTINATION\fR ] .SH "DESCRIPTION" .PP \fBltfs_ordered_copy\fR is a program to copy files from source to destination with LTFS diff --git a/man/ltfsck.8 b/man/ltfsck.8 index 3961e343..35e510ff 100644 --- a/man/ltfsck.8 +++ b/man/ltfsck.8 @@ -1,5 +1,5 @@ .\" auto-generated by docbook2man-spec from docbook-utils package -.TH "LTFSCK" "8" "16 July 2020" "IBM Spectrum Archive" "IBM Spectrum Archive Command Reference" +.TH "LTFSCK" "8" "20 January 2022" "LTFS" "LTFS Command Reference" .SH NAME ltfsck \- Recover and rollback utility for LTFS formatted tape .SH SYNOPSIS @@ -11,7 +11,7 @@ On Linux, \fIdevice_name\fR is like .SH "DESCRIPTION" .PP \fBltfsck\fR is a program to recover an inconsistent -LTFS formatted medium and roll back utility of the IBM Spectrum Archive. +LTFS formatted medium and roll back utility of the LTFS. .SH "OPTIONS" .PP These programs follow the usual GNU command line syntax, @@ -90,12 +90,13 @@ Use the specified key manager interface backend (default: none) \fB-x, --fulltrace\fR Enable full function call tracing (slow) .TP -\fB--capture-index\fR -Capture index information to the current directory -(-g is effective for this option) +\fB--capture-index=\fIdir\fB\fR +Capture indexes read successfully to the specified directory by dir. (-g is effective for this option) +File name of each index is [BARCODE]-[GEN]-[PARTITION].xml if tape serial (barcode) +is specified at format time. Otherwise it is [VOL_UUID]-[GEN]-[PARTITION].xml. .TP \fB--salvage-rollback-points\fR List the rollback points of the cartridge that has no EOD .SH "SEE ALSO" .PP -ltfs-sde(8), mkltfs(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). +ltfs(8), mkltfs(8), ltfsindextool(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). diff --git a/man/ltfsindextool.8 b/man/ltfsindextool.8 new file mode 100644 index 00000000..dd932c29 --- /dev/null +++ b/man/ltfsindextool.8 @@ -0,0 +1,72 @@ +.\" auto-generated by docbook2man-spec from docbook-utils package +.TH "LTFSINDEXTOOL" "8" "19 August 2021" "LTFS" "LTFS Command Reference" +.SH NAME +ltfsindextool \- Low level index checking tool for LTFS +.SH SYNOPSIS +.sp +\fBltfsindextool\fR [ \fB-d \fIname\fB\fR ] [ \fB-p \fIpart_num\fB\fR ] [ \fB-s \fIblock\fB\fR ] [ \fB-b \fInum\fB\fR ] [ \fB-i \fIfile\fB\fR ] [ \fB-e \fIname\fB\fR ] [ \fB-q\fR ] [ \fB-t\fR ] [ \fB-V\fR ] [ \fB-h\fR ] [ \fB-p\fR ] [ \fB\fIfilename\fB\fR ] +.SH "DESCRIPTION" +.PP +\fBltfsindextool\fR is a low level index checking tool. +.PP +There are 2 features. One is captureing indexes on a tape, the other is checking captured index from a file. +The command runs as index checking mode when filename is specified. It runs as index capturing mode when +-d option is specified. It runs as index checking mode when both filename and -d option are specified. +-p, -s --output-dir, -b is available only when it runs under index captureing mode. They would be ignored +when it runs under index checking mode. +.SH "OPTIONS" +.PP +These programs follow the usual GNU command line syntax, +with long options starting with two dashes ('-'). A summary of +options is included below. For a complete description, see the +\fBInfo\fR files. +.TP +\fB-d, --device=\fIname\fB\fR +Tape device name to capture indexes. On Linux, \fIname\fR is like +\&'/dev/IBMtape0', on OSX, \fIname\fR is like '0'. +.TP +\fB-p, --partition=\fIpart_num\fB\fR +Partition to capture indexes. Shall be 0 or 1. Capture indexes on both partitions +.TP +\fB-s, --start-pos=\fIblock\fB\fR +Block number to start capturing indexes +.TP +\fB--output-dir=\fIdir\fB\fR +Directory to store captured indexes +.TP +\fB-b, --blocksize=\fInum\fB\fR +Specify the LTFS record size +.TP +\fB-i, --config=\fIname\fB\fR +Use the specified configuration file +.TP +\fB-e, --backend=\fIname\fB\fR +Use the specified tape device backend +.TP +\fB--kmi-backend=\fIname\fB\fR +Use the specified key manager interface backend (default: none) +.TP +\fB-q, --quiet\fR +Suppress progress information and general messages +.TP +\fB-t, --trace\fR +Enable function call tracing +.TP +\fB-V, --version\fR +Version information +.TP +\fB-h, --help\fR +Show help information +.SS "USAGE EXAMPLE" +.sp +.nf + ltfsindextool -d /dev/sg10 + ltfsindextool -d /dev/sg10 -p 1 --output-dir=/foo + ltfsindextool -d ltfs-index-1-35.xml + +.sp +.fi +.PP +.SH "SEE ALSO" +.PP +ltfs(8), mkltfs(8), ltfsck(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). diff --git a/man/mkltfs.8 b/man/mkltfs.8 index 37e59e28..e3b87f73 100644 --- a/man/mkltfs.8 +++ b/man/mkltfs.8 @@ -1,14 +1,14 @@ .\" auto-generated by docbook2man-spec from docbook-utils package -.TH "MKLTFS" "8" "16 July 2020" "IBM Spectrum Archive" "IBM Spectrum Archive Command Reference" +.TH "MKLTFS" "8" "08 February 2022" "LTFS" "LTFS Command Reference" .SH NAME mkltfs \- Format a tape in the drive to LTFS format .SH SYNOPSIS .sp -\fBmkltfs\fR \fB-d \fIname\fB\fR [ \fB-f\fR ] [ \fB-s \fIid\fB\fR ] [ \fB-n \fIname\fB\fR ] [ \fB-r \fIrules\fB\fR ] [ \fB-w\fR ] [ \fB-q\fR ] [ \fB-t\fR ] [ \fB-V\fR ] [ \fB-h\fR ] [ \fB-p\fR ] +\fBmkltfs\fR \fB-d \fIname\fB\fR [ \fB-f\fR ] [ \fB-s \fIid\fB\fR ] [ \fB-n \fIname\fB\fR ] [ \fB-r \fIrules\fB\fR ] [ \fB-w\fR ] [ \fB-q\fR ] [ \fB-t\fR ] [ \fB-V\fR ] [ \fB-h\fR ] [ \fB-p\fR ] .SH "DESCRIPTION" .PP \fBmkltfs\fR is a program to format a media for use with -the IBM Spectrum Archive. +the LTFS. .SH "OPTIONS" .PP These programs follow the usual GNU command line syntax, @@ -75,7 +75,7 @@ Full help, including advanced options /home/piste/ltfs05-sde/bin/mkltfs --device=/dev/IBMtape0 --rules="size=100K" /home/piste/ltfs05-sde/bin/mkltfs --device=/dev/IBMtape0 --rules="size=1M/name=*.jpg" /home/piste/ltfs05-sde/bin/mkltfs --device=/dev/IBMtape0 --rules="size=1M/name=*.jpg:*.png" - + .sp .fi .PP @@ -112,6 +112,9 @@ Enable full function call tracing (slow) \fB--long-wipe\fR Unformat the medium and erase any data on the tape by overwriting special data pattern. This operation takes over 3 hours. Once you start, you cannot interrupt it. +.TP +\fB--destructive\fR +Use destructive format/unformat. This operation takes longer time in the LTO9 drive or later because of the media optimization procedure. .SH "SEE ALSO" .PP -ltfs-sde(8), ltfsck(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). +ltfs(8), ltfsck(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). diff --git a/man/sgml/ltfs-sde.sgml b/man/sgml/ltfs.sgml similarity index 93% rename from man/sgml/ltfs-sde.sgml rename to man/sgml/ltfs.sgml index 9fc854b6..9d754ac2 100644 --- a/man/sgml/ltfs-sde.sgml +++ b/man/sgml/ltfs.sgml @@ -1,10 +1,10 @@ + GNU"> ]> - IBM Spectrum Archive Command Reference + LTFS Command Reference @@ -12,7 +12,7 @@ &dhcommand; 8 - IBM Spectrum Archive + LTFS @@ -156,9 +156,12 @@ - + - Attempt to mount on previous index generation (read-only mount) + Attempt to mount on previous index generation on tape or specified index file by read-only mode + + When both index_file and device name is specified on -o devname option, normal read-only mount is attempted. + When index_file is only specified, meta-data read-only mode is attempted. EAGAIN would be returned at accessing contents of file. @@ -356,9 +359,13 @@ - + - Capture latest index to work directory at unmount + + Capture index to the specified directory by dir when index is updated. + File name of each index is [BARCODE]-[GEN]-[PARTITION].xml if tape serial (barcode) + is specified at format time. Otherwise it is [VOL_UUID]-[GEN]-[PARTITION].xml. + diff --git a/man/sgml/ltfs_ordered_copy.sgml b/man/sgml/ltfs_ordered_copy.sgml index e56d73c8..fed9994f 100644 --- a/man/sgml/ltfs_ordered_copy.sgml +++ b/man/sgml/ltfs_ordered_copy.sgml @@ -4,7 +4,7 @@ ]> - IBM Spectrum Archive Command Reference + LTFS Command Reference @@ -12,7 +12,7 @@ &dhcommand; 1 - IBM Spectrum Archive + LTFS diff --git a/man/sgml/ltfsck.sgml b/man/sgml/ltfsck.sgml index ac28a61c..0e9bcd08 100644 --- a/man/sgml/ltfsck.sgml +++ b/man/sgml/ltfsck.sgml @@ -4,7 +4,7 @@ M - IBM Spectrum Archive Command Reference + LTFS Command Reference @@ -12,7 +12,7 @@ M&dhcommand; 8 - IBM Spectrum Archive + LTFS @@ -51,7 +51,7 @@ MDESCRIPTION &dhcommand; is a program to recover an inconsistent - LTFS formatted medium and roll back utility of the IBM Spectrum Archive. + LTFS formatted medium and roll back utility of the LTFS. @@ -209,11 +209,12 @@ M - + - Capture index information to the current directory - (-g is effective for this option) + Capture indexes read successfully to the specified directory by dir. (-g is effective for this option) + File name of each index is [BARCODE]-[GEN]-[PARTITION].xml if tape serial (barcode) + is specified at format time. Otherwise it is [VOL_UUID]-[GEN]-[PARTITION].xml. @@ -228,7 +229,7 @@ M SEE ALSO - ltfs-sde(8), mkltfs(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). + ltfs(8), mkltfs(8), ltfsindextool(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). diff --git a/man/sgml/ltfsindextool.sgml b/man/sgml/ltfsindextool.sgml new file mode 100644 index 00000000..d4530105 --- /dev/null +++ b/man/sgml/ltfsindextool.sgml @@ -0,0 +1,159 @@ + + GNU"> + ]> + + + LTFS Command Reference + + + + + &dhcommand; + + 8 + LTFS + + + + &dhcommand; + Low level index checking tool for LTFS + + + + + &dhcommand; + + + + + + + + + + + + filename + + + + + DESCRIPTION + + &dhcommand; is a low level index checking tool. + + + There are 2 features. One is captureing indexes on a tape, the other is checking captured index from a file. + The command runs as index checking mode when filename is specified. It runs as index capturing mode when + -d option is specified. It runs as index checking mode when both filename and -d option are specified. + -p, -s --output-dir, -b is available only when it runs under index captureing mode. They would be ignored + when it runs under index checking mode. + + + + + OPTIONS + + These programs follow the usual &gnu; command line syntax, + with long options starting with two dashes ('-'). A summary of + options is included below. For a complete description, see the + Info files. + + + + + + + + Tape device name to capture indexes. On Linux, name is like + '/dev/IBMtape0', on OSX, name is like '0'. + + + + + + + Partition to capture indexes. Shall be 0 or 1. Capture indexes on both partitions + + + + + + Block number to start capturing indexes + + + + + + Directory to store captured indexes + + + + + + Specify the LTFS record size + + + + + + Use the specified configuration file + + + + + + Use the specified tape device backend + + + + + + Use the specified key manager interface backend (default: none) + + + + + + Suppress progress information and general messages + + + + + + Enable function call tracing + + + + + + Version information + + + + + + Show help information + + + + + + USAGE EXAMPLE + + ltfsindextool -d /dev/sg10 + ltfsindextool -d /dev/sg10 -p 1 --output-dir=/foo + ltfsindextool -d ltfs-index-1-35.xml + + + + + + SEE ALSO + ltfs(8), mkltfs(8), ltfsck(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). + + + + + diff --git a/man/sgml/mkltfs.sgml b/man/sgml/mkltfs.sgml index f7d0aa31..6304945a 100644 --- a/man/sgml/mkltfs.sgml +++ b/man/sgml/mkltfs.sgml @@ -4,7 +4,7 @@ ]> - IBM Spectrum Archive Command Reference + LTFS Command Reference @@ -12,7 +12,7 @@ &dhcommand; 8 - IBM Spectrum Archive + LTFS @@ -41,7 +41,7 @@ DESCRIPTION &dhcommand; is a program to format a media for use with - the IBM Spectrum Archive. + the LTFS. @@ -228,12 +228,20 @@ + + + + + Use destructive format/unformat. This operation takes longer time in the LTO9 drive or later because of the media optimization procedure. + + + SEE ALSO - ltfs-sde(8), ltfsck(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). + ltfs(8), ltfsck(8), tape-backend(4), kmi-backend(4), ltfs.conf(5). diff --git a/messages/Makefile.am b/messages/Makefile.am index 3d712e00..26bc61dc 100644 --- a/messages/Makefile.am +++ b/messages/Makefile.am @@ -49,6 +49,7 @@ RESOURCES = \ libbin_mkltfs_dat.a \ libbin_ltfsck_dat.a \ + libbin_ltfsindextool_dat.a \ libbin_ltfs_dat.a \ libkmi_simple_dat.a \ libkmi_flatfile_dat.a \ diff --git a/messages/bin_ltfs/root.txt b/messages/bin_ltfs/root.txt index 91d19bd7..043aea02 100644 --- a/messages/bin_ltfs/root.txt +++ b/messages/bin_ltfs/root.txt @@ -3,7 +3,7 @@ // OO_Copyright_BEGIN // // -// Copyright 2010, 2020 IBM Corp. All rights reserved. +// Copyright 2010, 2022 IBM Corp. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -52,7 +52,7 @@ root:table { //unused 14010E:string { "Cannot load I/O scheduler \'%s\'." } 14011E:string { "Cannot allocate LTFS volume structure." } 14012E:string { "Tape backend option parsing failed." } - 14013E:string { "Cannot mount the volume." } + 14013E:string { "Cannot mount the volume from %s." } //unused 14014E:string { "Cannot duplicate index criteria." } 14015W:string { "Volume does not allow index criteria override. Ignoring user-specified criteria." } 14016E:string { "Cannot format data placement rules (%d)." } @@ -152,7 +152,14 @@ root:table { 14113I:string { "Specified mount point is listed if succeeded." } 14114E:string { "Cannot initialize the open file table." } 14115E:string { "Invalid scsi_append_only_mode option: %s." } - 14116E:string { "This medium is not supported (%d)." } + 14116E:string { "This medium is not supported (%d)." } + 14117I:string { "Metadata only rollback mount is specified. Mounting as meta read-only from %s." } + 14118E:string { "Unexpected option is specified for read only mount (%d)." } + 14119I:string { "Rollback mount from index file is specified. Mounting as read-only from %s." } + 14120W:string { "Cannot access to directory %s, disabled index capture mode (%d)." } + 14121I:string { "Index will be captured at %s at update" } + 14122I:string { "Index will not be captured." } + 14123W:string { "The main function of FUSE returned error (%d)." } // 14150 - 14199 are reserved for LE+ @@ -209,7 +216,7 @@ root:table { // Reserved 14434E // Reserved 14435I 14436I:string { " -o device_list Show available tape devices" } - 14437I:string { " -o rollback_mount= Attempt to mount on previous index generation (read-only mount)" } + 14437I:string { " -o rollback_mount= Attempt to mount on previous index generation on tape or specified index file (read-only mount)" } // Reserved 14438I 14439I:string { " -o noeject Do not eject the cartridge after unmount (default)" } 14440I:string { " -o noatime Do not update index if only access times have changed (default)" } @@ -227,7 +234,7 @@ root:table { // Reserved 14451I // Reserved 14454I 14455I:string { " -o kmi_backend= Key manager interface implementation to use (default: %s, use \"none\" to disable)" } - 14456I:string { " -o capture_index Capture latest index to work directory at unmount" } + 14456I:string { " -o capture_index= Capture index to the specified directory by dir when index is updated" } // Reserved 14457I // Reserved 14458I // Reserved 14459I diff --git a/messages/bin_ltfsck/root.txt b/messages/bin_ltfsck/root.txt index 16a6b4ee..4c5cc797 100644 --- a/messages/bin_ltfsck/root.txt +++ b/messages/bin_ltfsck/root.txt @@ -3,7 +3,7 @@ // OO_Copyright_BEGIN // // -// Copyright 2010, 2020 IBM Corp. All rights reserved. +// Copyright 2010, 2022 IBM Corp. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -151,6 +151,10 @@ root:table { 16109E:string { "This operation is not allowed on this medium (%s)." } 16110E:string { "The --salvage-rollback-points option was specified against a normal cartridge." } 16111I:string { "The recovery process is skipped because of a locked cartridge (%d)." } + 16112W:string { "Cannot rename %s to %s (%d)." } + 16113W:string { "Cannot access to directory %s, disabled index capture mode (%d)." } + 16114I:string { "Index will be captured in %s." } + 16115I:string { "Index will not be captured." } // Help messages 16400I:string { "Usage: %s [options] filesys" } @@ -180,7 +184,7 @@ root:table { " (Must be used for a cartridge that cannot be recovered by a normal option.)" } 16422I:string { " -m, --full-index-info Display full index information (Effective only for -l option)" } 16423I:string { " --kmi-backend= Override the default key manager interface backend" } - 16424I:string { " --capture-index Capture index information to the current directory (-g is effective for this option)" } + 16424I:string { " --capture-index= Capture indexes to the specified directory (-g is effective for this option)" } 16425I:string { " --syslogtrace Enable diagnostic output to stderr and syslog" } 16426I:string { " -V, --version Version information" } 16427I:string { " --salvage-rollback-points List the rollback points of the cartridge that has no EOD" } diff --git a/messages/bin_ltfsindextool/en.txt b/messages/bin_ltfsindextool/en.txt new file mode 100644 index 00000000..44064bb4 --- /dev/null +++ b/messages/bin_ltfsindextool/en.txt @@ -0,0 +1,39 @@ +// +// +// OO_Copyright_BEGIN +// +// +// Copyright 2010, 2020 IBM Corp. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// +// OO_Copyright_END +// + +en:table { + // This resource intentionally left blank. +} + diff --git a/messages/bin_ltfsindextool/en_US.txt b/messages/bin_ltfsindextool/en_US.txt new file mode 100644 index 00000000..ffd90dd2 --- /dev/null +++ b/messages/bin_ltfsindextool/en_US.txt @@ -0,0 +1,39 @@ +// +// +// OO_Copyright_BEGIN +// +// +// Copyright 2010, 2020 IBM Corp. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// +// OO_Copyright_END +// + +en_US:table { + // This resource intentionally left blank. +} + diff --git a/messages/bin_ltfsindextool/root.txt b/messages/bin_ltfsindextool/root.txt new file mode 100644 index 00000000..b84dc906 --- /dev/null +++ b/messages/bin_ltfsindextool/root.txt @@ -0,0 +1,109 @@ +// +// +// OO_Copyright_BEGIN +// +// +// Copyright 2010, 2020 IBM Corp. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// +// OO_Copyright_END +// + +// Messages for ltfsindextool. +root:table { + messages:table { + start_id:int { 19500 } + end_id: int { 19989 } + 19500I:string { "Starting ltfsindextool, %s version %s, log level %d." } + 19501E:string { "Cannot allocate LTFS volume structure." } + 19502I:string { "%s." } + 19503I:string { "GCC version is %s." } + 19504I:string { "Capture all indexes from both partitions." } + 19505I:string { "Capture all indexes from (%u, %llu)." } + 19506D:string { "Opening the device." } + 19507D:string { "Device opened." } + 19508E:string { "Cannot open backend \'%s\'." } + 19509E:string { "Cannot open key manager interface backend \'%s\'." } + 19510E:string { "Cannot open device \'%s\' (%d)." } + 19511E:string { "Could not initialize the key manager interface plug-in. \'%s\' (%d)." } + 19512E:string { "Key manager interface backend option parsing failed." } + 19513E:string { "Tape backend option parsing failed." } + 19514E:string { "Unknown option '%s %s'." } + 19515E:string { "Cannot set up tape device." } + 19516E:string { "Cannot allocate the read buffer." } + 19517E:string { "Cannot seek to the start position (%u, %llu, %d)." } + 19518E:string { "Cannot get the current position (%d)." } + 19519E:string { "Cannot read a block (%u, %llu, %ld)." } + 19520D:string { "Closing the device." } + 19521W:string { "Cannot unload backend." } + 19522D:string { "Device closed." } + 19523I:string { "Failed to capture index (%d)." } + 19524I:string { "Captured indexes successfully." } + 19525D:string { "Validating command line options." } + 19526E:string { "File name to check or device name to capture must be specified." } + 19527D:string { "Command line options are valid." } + 19528W:string { "Cannot unload key manager interface backend." } + 19529I:string { "Reading an index from (%u, %llu)." } + 19530I:string { "Block in (%u, %llu) does not look an index, seek to next position (%d)." } + 19531E:string { "Cannot seek to the next position (%u, %llu, %d)." } + 19532E:string { "Cannot allocate the file name." } + 19533E:string { "Cannot open the file %s (%d)." } + 19534I:string { "Reached to EOD (%u, %llu)." } + 19535I:string { "Reached to EOD but cannot get the position (%d)." } + 19536E:string { "Cannot write a block (%ld, %d)." } + 19537I:string { "Detected the end of the index (%u, %llu)." } + 19538E:string { "Detected the EOD in the middle of index (%u, %llu)." } + 19539I:string { "Wrote an index, length is %llu." } + 19540E:string { "Partition number must be 0 or 1." } + 19541E:string { "Operation mode is wrong." } + 19542I:string { "Launched by \"%s\"." } + 19543I:string { "Checking the index file \"%s\"." } + 19544I:string { "Checked the index successfully." } + 19545E:string { "Failed to check the index (%d)." } + 19546I:string { "%s version %s." } + 19547I:string { "Creating an index file: %s" } + 19548E:string { "Start position shall be 5 or larger (%llu)." } + + // Help messages + 19900I:string { "Usage: %s [options] [filename]" } + 19901I:string { "Available options are:" } + 19902I:string { " -d, --device= Tape device (Capture index to specified file when this option is specified. Otherwise run check process with specified file)" } + 19903I:string { " -p, --partition= Partition number 0 or 1, capture both partitions if not specified" } + 19904I:string { " -s, --start-pos= Block number to start capturing (default: %d)" } + 19905I:string { " --output-dir= Directory to store captured indexes (default: \'%s\')" } + 19906I:string { " -b, --blocksize= Specify the LTFS record size (default: %d)" } + 19907I:string { " -i, --config= Use the specified configuration file (default: %s)" } + 19908I:string { " -e, --backend= Use the specified tape device backend (default: %s)" } + 19909I:string { " --kmi-backend= Use the specified key manager interface backend (default: %s)" } + 19910I:string { " -q, --quiet Suppress progress information and general messages" } + 19911I:string { " -t, --trace Enable function call tracing" } + 19912I:string { " -V, --version Version information" } + 19913I:string { " -h, --help This help" } + 19914I:string { "Usage example:" } + 19915I:string { " %s --device=%s -p=%d" } + } +} diff --git a/messages/bin_mkltfs/root.txt b/messages/bin_mkltfs/root.txt index 245bd7bd..7923a0de 100644 --- a/messages/bin_mkltfs/root.txt +++ b/messages/bin_mkltfs/root.txt @@ -127,5 +127,6 @@ root:table { 15422I:string { " --syslogtrace Enable diagnostic output to stderr and syslog" } 15423I:string { " -V, --version Version information" } 15424I:string { " --long-wipe Unformat the medium and erase any data on the tape by overwriting special data pattern.\n This operation takes over 3 hours. Once you start, you cannot interrupt it" } + 15425I:string { " --destructive Use destructive format/unformat. This operation takes longer time\n in the LTO9 drive or later because of the media optimization procedure" } } } diff --git a/messages/internal_error/root.txt b/messages/internal_error/root.txt index 685b26dc..7692c8b9 100644 --- a/messages/internal_error/root.txt +++ b/messages/internal_error/root.txt @@ -251,6 +251,65 @@ root:table { I1198E:string{ "The tape is already removed." } I1199E:string{ "You need to move tape to a storage slot first before the operation." } I1200E:string{ "The operation needs to be started again." } + I1201E:string{ "Locate returns write-perm error." } + I1202E:string{ "Failed to open a stats DB." } + I1203E:string{ "There is no trailing FM after an index." } + I1204E:string{ "Failed to update safename." } + I1205E:string{ "Unwritten file contents exists in sync." } + + // Error codes for the XML parser + I5000E:string{ "Error happens in reading XML node." } + I5001E:string{ "Error happens in reading XML const value." } + I5002E:string{ "Unexpected node type is detected." } + I5003E:string{ "Unexpected XML document end is detected." } + I5004E:string{ "Cannot detect a tags is empty or not." } + I5005E:string{ "The tag is empty." } + I5006E:string{ "Error skipping unrecognized tag." } + I5007E:string{ "Do not find required tag." } + I5008E:string{ "Duplicated tag is detected." } + I5009E:string{ "XML tag is not closed correctly." } + I5010E:string{ "Cannot save the tag." } + I5011E:string{ "Unexpected top tag in the XML document." } + I5012E:string{ "Unexpected encoding is detected." } + I5013E:string{ "Failed to get attribute of the top tag." } + I5014E:string{ "Wrong UUID is detected." } + I5015E:string{ "Cannot parse generation number correctly." } + I5016E:string{ "Cannot parse index's updatetime correctly." } + I5017E:string{ "Wrong tape position of index or labal is detected." } + I5018E:string{ "Wrong tape position of previous index is detected." } + I5019E:string{ "Unexpected policyupdate value is detected." } + I5020E:string{ "Wrong policy value is detected." } + I5021E:string{ "Too long index comment." } + I5022E:string{ "Wrong maxuid tag is detected." } + I5023E:string{ "Unexpected readonly value is detected in a dir." } + I5024E:string{ "Unexpected timestamp in dir mtime." } + I5025E:string{ "Unexpected timestamp in dir creation time." } + I5026E:string{ "Unexpected timestamp in dir atime." } + I5027E:string{ "Unexpected timestamp in dir ctime." } + I5028E:string{ "Unexpected timestamp in dir backup time." } + I5029E:string{ "Unexpected xattr type." } + I5030E:string{ "Unexpected xattr size." } + I5031E:string{ "Unexpected UID number." } + I5032E:string{ "UID number validation is failed." } + I5033E:string{ "Unexpected readonly value is detected in a file." } + I5034E:string{ "Unexpected timestamp in file mtime." } + I5035E:string{ "Unexpected timestamp in file creation time." } + I5036E:string{ "Unexpected timestamp in file atime." } + I5037E:string{ "Unexpected timestamp in file ctime." } + I5038E:string{ "Unexpected timestamp in file backup time." } + I5039E:string{ "Unexpected file size value." } + I5040E:string{ "Unexpected partition ID." } + I5041E:string{ "Unexpected startblock." } + I5042E:string{ "Unexpected offset." } + I5043E:string{ "Unexpected bytecount." } + I5044E:string{ "Unexpected fileoffset." } + I5045E:string{ "Extent list is overlapped." } + I5046E:string{ "Extent list is longer than file size." } + I5047E:string{ "Unexpected timestamp in tape format time in a label." } + I5048E:string{ "Unexpected partition map in a label." } + I5049E:string{ "Unexpected blocksize in a label." } + I5050E:string{ "Unexpected compression in a label." } + I5051E:string{ "Unsupported index type is specified." } // Special error codes I9997E:string{ "Child process error (ltfsck/mkltfs): %s (%d)." } diff --git a/messages/iosched_unified/root.txt b/messages/iosched_unified/root.txt index 3d8f4e0c..a2ecb64e 100644 --- a/messages/iosched_unified/root.txt +++ b/messages/iosched_unified/root.txt @@ -62,7 +62,8 @@ root:table { 13022W:string { "Freeing a dentry priv with outstanding write requests. This is a bug." } //unused 13023W:string { "Failed to copy a request: will stop writing file to the index partition." } 13024I:string { "Clean up extents and append index at index partition (%d)." } - 13025I:string { "Get error position (%d, %d)." } + 13025I:string { "Truncate extents larger than position (%d, %lld), block size = %ld." } 13026E:string { "Write perm handling error : %s (%d)." } + 13027I:string { "Error position is larger than last index position: (%d, %lld), last index = %lld." } } } diff --git a/messages/libltfs/root.txt b/messages/libltfs/root.txt index eb867875..a4f0515b 100644 --- a/messages/libltfs/root.txt +++ b/messages/libltfs/root.txt @@ -3,7 +3,7 @@ // OO_Copyright_BEGIN // // -// Copyright 2010, 2020 IBM Corp. All rights reserved. +// Copyright 2010, 2022 IBM Corp. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -97,7 +97,7 @@ root:table { 11002E:string { "Cannot instantiate LTFS volume: failed to allocate index data." } 11003E:string { "Cannot retrieve device capacity data (%d)." } 11004E:string { "Cannot take the device lock (%s)." } - 11005I:string { "Mounting the volume." } + 11005I:string { "Mounting the volume from %s." } 11006E:string { "Cannot read volume: failed to load the tape." } 11007D:string { "Tape is loaded." } 11008D:string { "Reading partition labels." } @@ -124,7 +124,7 @@ root:table { 11029E:string { "Cannot mount volume: failed to save the append position for the index partition." } 11030I:string { "Failed to sync volume (%d). Stop the periodic sync thread." } //unused 11030E:string { "Cannot release the device lock (%s)." } - //11031D:string { "Volume mounted successfully." } + 11031I:string { "Volume mounted successfully. %s : Gen = %lld / (%c, %lld) -> (%c, %lld) / %s." } 11032D:string { "Unmounting the volume." } 11033E:string { "Cannot unmount: failed to write an index." } 11034I:string { "Volume unmounted successfully." } @@ -237,7 +237,7 @@ root:table { 11140E:string { "Cannot remove extended attribute: failed to look up the xattr (%d)." } 11141E:string { "Cannot list physical xattrs: failed to generate namespace prefix (%d)." } 11142E:string { "Cannot list physical xattrs: failed to convert key to system locale (%d)." } - //unused 11143E + 11143E:string { "Specified length of buffer is too short. (%lu)" } //unused 11144E:string { "Cannot list physical xattrs: failed to convert key to system locale (%d)." } 11145E:string { "Cannot get attribute %s: failed to generate the time string." } 11146E:string { "Invalid index criteria option \'%s\'." } @@ -427,9 +427,9 @@ root:table { 11330I:string { "Loading cartridge." } 11331E:string { "Failed to load the cartridge (%s)." } 11332I:string { "Load successful." } - 11333I:string { "A cartridge with write-perm error is detected on %s. Seek the newest index (IP Gen = %llu, VCR = %llu) (DP Gen = %llu, VCR = %llu) (VCR = %llu)." } + 11333I:string { "A cartridge with write-perm error is detected on %s. Seek the newest index (IP: Gen = %llu, VCR = %llu) (DP: Gen = %llu, VCR = %llu) (VCR = %llu)." } 11334I:string { "Remove extent : %s (%llu, %llu)." } - 11335D:string { "Get physical block position (%d - %d)." } + //unused 11335D:string { "Get physical block position (%d - %d)." } 11336I:string { "The attribute does not exist. Ignore the expected error." } 11337I:string { "Update index-dirty flag (%d) - %s (0x%p)." } 11338I:string { "Syncing index of %s %s." } @@ -559,13 +559,13 @@ v 17007E:string { "Cannot instantiate an LTFS label parser for file \'%s\'." } 17008E:string { "Cannot parse XML label from file \'%s\'." } 17009E:string { "Cannot instantiate an LTFS label parser for a memory buffer." } - 17010E:string { "Cannot parse XML label from memory." } + 17010E:string { "Cannot parse XML label from memory (%d)." } 17011E:string { "Cannot instantiate an index parser for file \'%s\'." } - 17012E:string { "Cannot parse index from file \'%s\'." } + 17012E:string { "Cannot parse index from file \'%s\' (%d)." } 17013E:string { "Cannot parse index: failed to determine medium position (%d)." } 17014E:string { "Cannot parse index: failed to create XML parser input buffer." } 17015E:string { "Cannot parse index: failed to create XML reader." } - 17016E:string { "Cannot parse index direct from medium." } + 17016E:string { "Cannot parse index direct from medium (%d)." } 17017E:string { "XML parser: unexpected top-level tag \'%s\'." } 17018E:string { "XML parser: unsupported encoding \'%s\'." } 17019E:string { "XML parser: no schema version found." } @@ -581,7 +581,7 @@ v 17029E:string { "XML parser: invalid UUID %s." } 17030E:string { "XML parser: failed to normalize %s \'%s\'." } 17031E:string { "XML parser: invalid %s \'%s\'." } - 17032E:string { "XML parser: compression must be \'true\' (1) or \'false\' (0)." } + 17032E:string { "XML parser: boolean must be \'true\' (1) or \'false\' (0) but it is \'%s\'." } 17033E:string { "XML parser: invalid partition \'%s\'." } 17034E:string { "XML parser: invalid time \'%s\' (%d)." } 17035E:string { "XML parser: expected a text node." } @@ -650,8 +650,8 @@ v 17098I:string { "Device Name = %s (%d.%d.%d.%d), Vendor ID = %s, Product ID = %s, Serial Number = %s, Product Name =%s." } 17099E:string { "Failed to spawn the periodic sync thread (%d)." } - 17100E:string { "XML parser: UID on the root directory must be 1." } - 17101E:string { "XML parser: UID 1 is reserved for the root directory." } + 17100E:string { "XML parser: UID on the root directory must be 1" } + 17101E:string { "XML parser: UID 1 is reserved for the root directory (%s)." } 17102E:string { "Cannot set PEWS: Mode Sense for Device Configuration Extension failed (%d)." } 17103E:string { "Cannot set PEWS: Mode Select for Device Configuration Extension failed (%d)." } 17104E:string { "Cannot get PEWS: Mode Sense for Device Configuration Extension failed (%d)." } @@ -778,7 +778,7 @@ v 17233E:string { "Failed to kick gcore." } 17234W:string { "The index read from the tape uses a newer version of the LTFS format than the one supported by this software. If this tape is modified, the index downgrades to format version %s from %d.%d.%d." } 17235I:string { "Writing index of %s to %c (Reason: %s, %lld files) %s." } - 17236I:string { "Wrote index of %s (%c, %s)." } + 17236I:string { "Wrote index of %s (Gen = %lld, Part = %c, Pos = %lld, %s)." } 17237E:string { "WORM related error (%s)." } 17238I:string { "WORM status updated (%s=>%d) '%s'." } 17239E:string { "Failed to update density (%s) %d." } @@ -808,6 +808,41 @@ v 17264I:string { "The index on %s is newer, but MAM shows a permanent write error happened on %s." } 17265I:string { "Skip writing the index because of %s." } 17266I:string { "Skip setting the append only mode because the drive doesn't seem to support it." } + 17267E:string { "Locate command returns write-perm error (%d). Replace a return code to %d." } + 17268E:string { "XML parser: Detected invalid %s in a label." } + 17269W:string { "XML parser: Failed to parse a key name of xattr in %s. Skip one xattr." } + 17270E:string { "XML parser: Detected invalid %s in a file (%s)." } + 17271E:string { "Cannot save the conflicted info of symlink and extent: file %s." } + 17272E:string { "XML parser: Detected invalid %s in a directory (%s)." } + 17273E:string { "XML parser: Detected an I/O error on reading an index from the tape (%d)." } + 17274W:string { "XML parser: Unexpected error code is detected (%d)." } + 17275I:string { "RAO command returns error on %s (%d)." } + 17276I:string { "Failed to create RAO output file: %s (%d)." } + 17277I:string { "Failed to write to RAO output file: %s (%d)." } + 17278I:string { "Written length to RAO output file is unexpected: %s (Actual %ld, Expected %ld)." } + 17279I:string { "Failed to open RAO input file: %s (%d)." } + 17280I:string { "Failed to get the length of the RAO input file: %s (%d)." } + 17281I:string { "Failed to read from RAO input file: %s (%d)." } + 17282I:string { "Read length from RAO input file is unexpected: %s (Actual %ld, Expected %ld)." } + 17283I:string { "Detected unmatched VCR value between MAM and VCR (%llu, %llu)." } + 17284I:string { "Seaching the final index in %s." } + 17285E:string { "Failed to search the final index in %s (%d)." } + 17286I:string { "VCR value is matched between %s-MAM and VCR (%llu)." } + 17287I:string { "Making R/O mount from the location (%c, %llu)." } + 17288I:string { "Detected the final indexes (IP: Gen = %llu, Pos = %llu) (DP: Gen = %llu, Pos = %llu)." } + 17289I:string { "Skip parsing the final index on IP." } + 17290I:string { "Partitioning the medium with the destructive method." } + 17291I:string { "Unpartitioning the medium with the destructive method." } + 17292I:string { "Current position is (%llu, %llu), Error position is (%llu, %llu)." } + 17293E:string { "UUID in the index does not match the label." } + + 17300I:string { "Wrote inc-index of %s (Gen = %lld, Part = %c, Pos = %lld, %s)." } + 17301I:string { "Info inc-index, Gen = %lld, Full Part = %c, Full Pos = %lld, Back Part = %c, Back Pos = %lld)." } + 17302E:string { "Path helper: Provided path must be an absolute path (%s)." } + 17303E:string { "Unexpected value was found in the reason of inc-journal entry (%d)." } + 17304E:string { "Unexpected value was provided to _xml_write_incremental_delete (%d)." } + 17305E:string { "Failed to construct a path helper (push: %d)." } + 17306E:string { "Failed to find a corresponded directory in path helper (push: %d)." } // For Debug 19999I:string { "%s %s %d." } diff --git a/messages/make_message_src.sh b/messages/make_message_src.sh index 82488e81..13ca3861 100755 --- a/messages/make_message_src.sh +++ b/messages/make_message_src.sh @@ -78,18 +78,20 @@ make_obj() { NetBSD) # generate libtool archive for later linking mv lib${BASENAME}.a ../../lib${BASENAME}_dat.a - OBJFILE=${BASENAME}_dat.o - LTFILE=${BASENAME}_dat.lo mkdir -p .libs ../../.libs - mv ${OBJFILE} .libs - LTVERS=`libtool --version | - sed -e 's/^\([^ ]*\) (GNU \(.*\)) \(.*\)$$/\1 - GNU \2 \3/' -e q` - echo "# ${OBJFILE} - a libtool object file" > ${LTFILE} - echo "# Generated by ${LTVERS}" >> ${LTFILE} - echo "pic_object='.libs/${OBJFILE}'" >> ${LTFILE} - echo "non_pic_object=none" >> ${LTFILE} - libtool --mode=link --tag=CC cc -o ../../lib${BASENAME}_dat.la ${LTFILE} - cp ../../lib${BASENAME}_dat.a ../../.libs + LTFILES="" + LTVERS=`libtool --version | + sed -e 's/^\([^ ]*\) (GNU \(.*\)) \(.*\)$$/\1 - GNU \2 \3/' -e q` + for OBJFILE in *.o ; do + LTFILE=${OBJFILE%.o}.lo + mv ${OBJFILE} .libs + echo "# ${OBJFILE} - a libtool object file" > ${LTFILE} + echo "# Generated by ${LTVERS}" >> ${LTFILE} + echo "pic_object='.libs/${OBJFILE}'" >> ${LTFILE} + echo "non_pic_object=none" >> ${LTFILE} + LTFILES="${LTFILES} ${LTFILE}" + done + libtool --mode=link --tag=CC cc -o ../../lib${BASENAME}_dat.la ${LTFILES} ;; *) mv ${BASENAME}_dat.o ../../lib${BASENAME}_dat.a diff --git a/messages/print_error_messages.py b/messages/print_error_messages.py index fc5dc900..019fa820 100755 --- a/messages/print_error_messages.py +++ b/messages/print_error_messages.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python import os import re @@ -9,45 +9,45 @@ # List the messages defined in the message bundles for d, dirs, files in os.walk('.'): - if d == 'messages': - continue - if d == "messages/internal_error": - continue - msg_list = dict() - for f in files: - start_id = 0 - end_id = 1000000 - if re.search(r'\.txt$', f): - with file(os.path.join(d, f), 'r') as fd: - linenum = 1 - for line in fd: - m = re.search(r'start_id:int\s*{\s*(?P[0-9]+)\s*}', line) - if m is not None: - start_id = int(m.group('val')) - m = re.search(r'end_id:int\s*{\s*(?P[0-9]+)\s*}', line) - if m is not None: - end_id = int(m.group('val')) - if end_id < start_id: - print 'Warning: strange message ID range (%d-%d) in %s' % ( - start_id, end_id, os.path.join(d, f)) + if d == 'messages': + continue + if d == "messages/internal_error": + continue + msg_list = dict() + for f in files: + start_id = 0 + end_id = 1000000 + if re.search(r'\.txt$', f): + fd = open(os.path.join(d, f), 'r') + linenum = 1 + for line in fd: + m = re.search(r'start_id:int\s*{\s*(?P[0-9]+)\s*}', line) + if m is not None: + start_id = int(m.group('val')) + m = re.search(r'end_id:int\s*{\s*(?P[0-9]+)\s*}', line) + if m is not None: + end_id = int(m.group('val')) + if end_id < start_id: + print('Warning: strange message ID range (%d-%d) in %s' % ( + start_id, end_id, os.path.join(d, f))) - m = re.search(r'^(?P.*)//', line) - if m is not None: - m = re.search(re_msgid_bundle, m.group('val')) - else: - m = re.search(re_msgid_bundle, line) - if m is not None: - val = int(m.group('val')) - if val < start_id or val > end_id: - print 'Message ID %s out of range (%d-%d) at %s:%d' % ( - m.group('id'), start_id, end_id, os.path.join(d, f), linenum) - else: - msg_list[m.group('id')] = m.group('msg') - linenum += 1 - if len(msg_list) > 0: - msg_ids[os.path.basename(d)] = msg_list + m = re.search(r'^(?P.*)//', line) + if m is not None: + m = re.search(re_msgid_bundle, m.group('val')) + else: + m = re.search(re_msgid_bundle, line) + if m is not None: + val = int(m.group('val')) + if val < start_id or val > end_id: + print('Message ID %s out of range (%d-%d) at %s:%d' % ( + m.group('id'), start_id, end_id, os.path.join(d, f), linenum)) + else: + msg_list[m.group('id')] = m.group('msg') + linenum += 1 + if len(msg_list) > 0: + msg_ids[os.path.basename(d)] = msg_list # Find unused and nonexistent message IDs for module in msg_ids: for id in msg_ids[module]: - print "#define LTFS%s %s" % (id, msg_ids[module][id]) + print("#define LTFS%s %s" % (id, msg_ids[module][id])) diff --git a/messages/tape_iokit/root.txt b/messages/tape_iokit/root.txt index 91ec17fc..699a0fc7 100644 --- a/messages/tape_iokit/root.txt +++ b/messages/tape_iokit/root.txt @@ -115,6 +115,8 @@ root:table { 30868I:string { "Retry to reserve from key registration (%s)" } 30869W:string { "The drive is already reserved: %s (%s)" } 30870I:string { "Failed to get the cartridge status. The cartridge is not loaded." } + 30871W:string { "Failed to get medium type code: medium type check is skipped." } + 30872I:string { "Setting up timeout values from %s." } 30992D:string { "Backend %s %s." } 30993D:string { "Backend %s: %d %s." } @@ -123,8 +125,7 @@ root:table { 30996D:string { "Backend %s: %llu %s." } 30997D:string { "Backend %s: (%llu, %llu) %s." } 30998D:string { "Backend %s: (%llu, %llu) FM = %llu %s." } - 30999I:string { "iokit backend options:\n" - " -o devname= tape device (default=%s)\n" + 30999I:string { "iokit backend options:\n -o devname= tape device (default=%s)\n" " -o scsi_lbprotect= enable drive logical block protection (default=off)\n." } } } diff --git a/messages/tape_linux_lin_tape/root.txt b/messages/tape_linux_lin_tape/root.txt index 7845c4a6..780aa3e7 100644 --- a/messages/tape_linux_lin_tape/root.txt +++ b/messages/tape_linux_lin_tape/root.txt @@ -68,7 +68,7 @@ root:table { 30423I:string { "Opening a device through ibmtape driver (%s)." } 30424E:string { "%s: medium is already mounted or in use." } 30425I:string { "Cannot open device \'%s\' (%d)." } - 30426W:string { "Device \'%s\' must be opened in read-only mode." } + 30426W:string { "Device \'%s\' must be opened in read-write mode." } 30427I:string { "Cannot open device: inquiry failed (%d)." } 30428I:string { "Product ID is \'%s\'." } 30429I:string { "Vendor ID is %s." } diff --git a/messages/tape_linux_sg/root.txt b/messages/tape_linux_sg/root.txt index 9698160b..5bfc632b 100644 --- a/messages/tape_linux_sg/root.txt +++ b/messages/tape_linux_sg/root.txt @@ -100,6 +100,7 @@ root:table { 30258W:string { "Cannot retrieve drive dump: failed to read buffer (%d)." } 30259W:string { "Cannot retrieve drive dump: failed to write to dump file (%d)." } 30260W:string { "Cannot retrieve drive dump: wrote %d bytes out, expected %d." } + 30261W:string { "Cannot retrieve drive dump: failed to write to communicate with drive. Tried (%d) times." } 30261I:string { "Taking drive dump in buffer." } 30262I:string { "Forcing drive dump." } 30263I:string { "%s returns %s (%d) %s." } @@ -133,6 +134,10 @@ root:table { 30291I:string { "Changer %s was reserved from this node and can be reserved from the current path." } 30292I:string { "Changer %s was reserved from this node but failed to reserve from the current path." } 30293I:string { "Changer %s was reserved from another node (%s)." } + 30294I:string { "Setting up timeout values from %s." } + 30295I:string { "Have unstable TUR response, start over (Cur = %d, Prev = %d)." } + 30296I:string { "Capturing a stable TUR at line %d." } + 30297W:string { "Cannot retrieve drive dump: failed to communicate with drive. Tried (%d) times." } 30392D:string { "Backend %s %s." } 30393D:string { "Backend %s: %d %s." } diff --git a/src/.gitignore b/src/.gitignore index e645f6e0..294b8bd7 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1 +1,2 @@ ltfs +ltfsmsg.h diff --git a/src/Makefile.am b/src/Makefile.am index 8dd6fce0..7afe5ef7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -40,7 +40,7 @@ ltfs_SOURCES = main.c ltfs_fuse.c ltfs_DEPENDENCIES = libltfs/libltfs.la ../messages/libbin_ltfs_dat.a ltfs_CPPFLAGS = @AM_CPPFLAGS@ -I ../ltfs-sde/src -fPIC ltfs_LDADD = libltfs/libltfs.la -ltfs_LDFLAGS = @AM_LDFLAGS@ -L../messages -lbin_ltfs_dat +ltfs_LDFLAGS = @AM_LDFLAGS@ ../messages/libbin_ltfs_dat.a endif PLAT_DRV = @@ -77,7 +77,6 @@ SUBDIRS = libltfs $(GEN_DRV) $(PLAT_DRV) $(SUB) iosched utils noinst_HEADERS = \ libltfs/base64.h \ - libltfs/config_file.h \ libltfs/dcache.h \ libltfs/fs.h \ libltfs/index_criteria.h \ @@ -92,14 +91,13 @@ noinst_HEADERS = \ libltfs/pathname.h \ libltfs/periodic_sync.h \ libltfs/xattr.h \ - libltfs/xml.h \ libltfs/xml_libltfs.h \ - libltfs/arch/errormap.h \ libltfs/arch/filename_handling.h \ libltfs/arch/uuid_internal.h \ libltfs/arch/version.h nobase_pkginclude_HEADERS = \ + libltfs/config_file.h \ libltfs/dcache_ops.h \ libltfs/iosched_ops.h \ libltfs/kmi_ops.h \ @@ -119,9 +117,11 @@ nobase_pkginclude_HEADERS = \ libltfs/uthash_ext.h \ libltfs/plugin.h \ libltfs/tape.h \ + libltfs/xml.h \ libltfs/arch/signal_internal.h \ libltfs/arch/arch_info.h \ libltfs/arch/time_internal.h \ + libltfs/arch/errormap.h \ tape_drivers/ibm_tape.h \ tape_drivers/spc_op_codes.h \ tape_drivers/ssc_op_codes.h \ diff --git a/src/iosched/Makefile.am b/src/iosched/Makefile.am index 698a9505..d3887662 100644 --- a/src/iosched/Makefile.am +++ b/src/iosched/Makefile.am @@ -39,13 +39,13 @@ BASENAMES = libiosched-fcfs libiosched-unified AM_LIBTOOLFLAGS = --tag=disable-static libiosched_fcfs_la_SOURCES = fcfs.c -libiosched_fcfs_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../messages/ -liosched_fcfs_dat +libiosched_fcfs_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../messages/libiosched_fcfs_dat.a libiosched_fcfs_la_DEPENDENCIES = ../../messages/libiosched_fcfs_dat.a ../libltfs/libltfs.la libiosched_fcfs_la_LIBADD = ../libltfs/libltfs.la libiosched_fcfs_la_CPPFLAGS = @AM_CPPFLAGS@ -I .. libiosched_unified_la_SOURCES = unified.c cache_manager.c -libiosched_unified_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../messages/ -liosched_unified_dat +libiosched_unified_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../messages/libiosched_unified_dat.a libiosched_unified_la_DEPENDENCIES = ../../messages/libiosched_unified_dat.a ../libltfs/libltfs.la libiosched_unified_la_LIBADD = ../libltfs/libltfs.la libiosched_unified_la_CPPFLAGS = @AM_CPPFLAGS@ -I .. diff --git a/src/iosched/unified.c b/src/iosched/unified.c index 56a39a04..a7ac70ca 100644 --- a/src/iosched/unified.c +++ b/src/iosched/unified.c @@ -2250,6 +2250,7 @@ int _unified_write_index_after_perm(int write_ret, struct unified_data *priv) { int ret = 0; struct tc_position err_pos; + uint64_t last_index_pos = UINT64_MAX; unsigned long blocksize; if (!IS_WRITE_PERM(-write_ret)) { @@ -2263,21 +2264,35 @@ int _unified_write_index_after_perm(int write_ret, struct unified_data *priv) ltfsmsg(LTFS_ERR, 13026E, "update MAM", ret); blocksize = ltfs_get_blocksize(priv->vol); - ret = tape_get_physical_block_position(priv->vol->device, &err_pos); + + ret = tape_get_first_untransfered_position(priv->vol->device, &err_pos); if (ret < 0) { ltfsmsg(LTFS_ERR, 13026E, "get error pos", ret); return ret; } - ltfsmsg(LTFS_INFO, 13025I, (int)err_pos.block, (int)blocksize); + /* Check the err_pos is larger than the last index position of the partition */ + if (err_pos.partition == ltfs_part_id2num(priv->vol->label->partid_ip, priv->vol)) { + last_index_pos = priv->vol->ip_coh.set_id; + } else { + last_index_pos = priv->vol->dp_coh.set_id; + } + + if (last_index_pos > err_pos.block) { + ltfsmsg(LTFS_INFO, 13027I, (int)err_pos.partition, + (unsigned long long)err_pos.block, (unsigned long long)last_index_pos); + err_pos.block = last_index_pos + 1; + } + ltfsmsg(LTFS_INFO, 13025I, (int)err_pos.partition, (unsigned long long)err_pos.block, blocksize); ret = ltfs_fsraw_cleanup_extent(priv->vol->index->root, err_pos, blocksize, priv->vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 13026E, "extent cleanup", ret); return ret; } - ret = ltfs_write_index(ltfs_ip_id(priv->vol), SYNC_WRITE_PERM, priv->vol); + ltfs_set_commit_message_reason(SYNC_WRITE_PERM, priv->vol); + ret = ltfs_write_index(ltfs_ip_id(priv->vol), SYNC_WRITE_PERM, LTFS_FULL_INDEX, priv->vol); return ret; } diff --git a/src/kmi/Makefile.am b/src/kmi/Makefile.am index 726a3aae..7aa05d94 100644 --- a/src/kmi/Makefile.am +++ b/src/kmi/Makefile.am @@ -41,13 +41,13 @@ AM_LIBTOOLFLAGS = --tag=disable-static libkmi_simple_la_SOURCES = simple.c key_format_ltfs.c libkmi_simple_la_DEPENDENCIES = ../../messages/libkmi_simple_dat.a ../libltfs/libltfs.la libkmi_simple_la_LIBADD = ../libltfs/libltfs.la -libkmi_simple_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../messages -lkmi_simple_dat +libkmi_simple_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../messages/libkmi_simple_dat.a libkmi_simple_la_CPPFLAGS = @AM_CPPFLAGS@ -I .. -DKMI_SIMPLE libkmi_flatfile_la_SOURCES = flatfile.c key_format_ltfs.c libkmi_flatfile_la_DEPENDENCIES = ../../messages/libkmi_flatfile_dat.a ../libltfs/libltfs.la libkmi_flatfile_la_LIBADD = ../libltfs/libltfs.la -libkmi_flatfile_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../messages -lkmi_flatfile_dat +libkmi_flatfile_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../messages/libkmi_flatfile_dat.a libkmi_flatfile_la_CPPFLAGS = @AM_CPPFLAGS@ -I .. install-exec-hook: diff --git a/src/kmi/flatfile.c b/src/kmi/flatfile.c index 628e9c27..53735d49 100644 --- a/src/kmi/flatfile.c +++ b/src/kmi/flatfile.c @@ -198,7 +198,6 @@ int flatfile_get_key(unsigned char **keyalias, unsigned char **key, void * const if (ret < 0) { ltfsmsg(LTFS_ERR, 15552E); if (dk_list) { - memset(dk_list, 0, strlen((char *) dk_list)); free(dk_list); } return ret; @@ -210,7 +209,6 @@ int flatfile_get_key(unsigned char **keyalias, unsigned char **key, void * const /* * Cache DK and DKi for revalidation at tape drive POR * if (dk_list) { - * memset(dk_list, 0, strlen((char *) dk_list)); * free(dk_list); * dk_list = NULL; * } diff --git a/src/libltfs/LTFS-MIB.txt b/src/libltfs/LTFS-MIB.txt index 5f9f8227..2770eaec 100644 --- a/src/libltfs/LTFS-MIB.txt +++ b/src/libltfs/LTFS-MIB.txt @@ -8,9 +8,9 @@ ibmLTFS MODULE-IDENTITY LAST-UPDATED "201503200000Z" ORGANIZATION "International Business Machines Corp." CONTACT-INFO "IBM Support" - DESCRIPTION "This file defines IBM Spectrum Archive LE/SDE MIB. + DESCRIPTION "This file defines IBM Storage Archive LE/SDE MIB. Licensed Materials - Property of IBM - (C) Copyright IBM Corp. 2014, 2015" + (C) Copyright IBM Corp. 2014, 2023" ::= { ibmProd 248 } ibm OBJECT IDENTIFIER ::= { enterprises 2 } diff --git a/src/libltfs/Makefile.am b/src/libltfs/Makefile.am index 22f22fcb..9b9ab438 100644 --- a/src/libltfs/Makefile.am +++ b/src/libltfs/Makefile.am @@ -62,6 +62,7 @@ libltfs_la_SOURCES = \ config_file.c \ plugin.c \ periodic_sync.c \ + inc_journal.c \ arch/uuid_internal.c \ arch/filename_handling.c \ arch/time_internal.c \ @@ -71,7 +72,7 @@ libltfs_la_SOURCES = \ libltfs_la_DEPENDENCIES = ../../messages/liblibltfs_dat.a ../../messages/libinternal_error_dat.a ../../messages/libtape_common_dat.a libltfs_la_LIBADD = libltfs_la_CPPFLAGS = @AM_CPPFLAGS@ @AM_EXTRA_CPPFLAGS@ @AM_EXTRA_CPPFLAGS@ -I .. -libltfs_la_LDFLAGS = @AM_LDFLAGS@ -L../../messages -llibltfs_dat -linternal_error_dat -ltape_common_dat +libltfs_la_LDFLAGS = @AM_LDFLAGS@ ../../messages/liblibltfs_dat.a ../../messages/libinternal_error_dat.a ../../messages/libtape_common_dat.a install-data-local: if [ ! -d "$(DESTDIR)$(prefix)/share/snmp" ]; then \ diff --git a/src/libltfs/arch/arch_info.c b/src/libltfs/arch/arch_info.c index 1ee677b5..e0577593 100644 --- a/src/libltfs/arch/arch_info.c +++ b/src/libltfs/arch/arch_info.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2021 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,7 +48,7 @@ */ #include "libltfs/ltfs.h" -#ifndef mingw_PLATFORM +#ifdef HAVE_SYS_SYSCTL_H #include #endif #include @@ -112,18 +112,19 @@ void show_runtime_system_info(void) } strcat(path, "/etc/"); strcat(path, dent->d_name); - if(stat(path, &stat_rel) != -1 && S_ISREG(stat_rel.st_mode)) { - fd = open(path, O_RDONLY); - if( fd == -1) { - ltfsmsg(LTFS_WARN, 17088W); - } else { + fd = open(path, O_RDONLY); + if( fd == -1) { + ltfsmsg(LTFS_WARN, 17088W); + } else { + if (fstat(fd, &stat_rel) != -1 && S_ISREG(stat_rel.st_mode)) { memset(destribution, 0, sizeof(destribution)); read(fd, destribution, sizeof(destribution)); if((tmp = strchr(destribution, '\n')) != NULL) *tmp = '\0'; ltfsmsg(LTFS_INFO, 17089I, destribution); - close(fd); } + + close(fd); } free(path); } @@ -133,7 +134,7 @@ void show_runtime_system_info(void) return; } -#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) +#elif defined(HAVE_SYS_SYSCTL_H) { int mib[2]; size_t len; diff --git a/src/libltfs/arch/errormap.c b/src/libltfs/arch/errormap.c index 748f1bd1..8459c23a 100644 --- a/src/libltfs/arch/errormap.c +++ b/src/libltfs/arch/errormap.c @@ -286,7 +286,63 @@ static struct error_map fuse_error_list[] = { { LTFS_TAPE_REMOVED, "I1198E", EIDRM}, { LTFS_NEED_MOVE, "I1199E", EINVAL}, { LTFS_NEED_START_OVER, "I1200E", EINVAL}, - + { LTFS_LOCATE_ERROR, "I1201E", EIO}, + { LTFS_STATS_DB_OPEN, "I1202E", EIO}, + { LTFS_NO_TRAIL_FM, "I1203E", EINVAL}, + { LTFS_SAFENAME_FAIL, "I1204E", EINVAL}, + { LTFS_SYNC_FAIL_ON_DP, "I1205E", EIO}, + { LTFS_XML_READ_FAIL, "I5000E", EINVAL}, + { LTFS_XML_CONST_FAIL, "I5001E", EINVAL}, + { LTFS_XML_WRONG_NODE, "I5002E", EINVAL}, + { LTFS_XML_UNEXPECTED_EOF, "I5003E", EINVAL}, + { LTFS_XML_EMPTY_UNKNOWN, "I5004E", EINVAL}, + { LTFS_XML_EMPTY, "I5005E", EINVAL}, + { LTFS_XML_SKIP_FAIL, "I5006E", EINVAL}, + { LTFS_XML_NO_REQUIRED_TAG, "I5007E", EINVAL}, + { LTFS_XML_DUPLICATED_TAG, "I5008E", EINVAL}, + { LTFS_XML_OPEN_TAG, "I5009E", EINVAL}, + { LTFS_XML_SAVE_FAIL, "I5010E", EINVAL}, + { LTFS_XML_WRONG_TOPTAG, "I5011E", EINVAL}, + { LTFS_XML_WRONG_ENCODING, "I5012E", EINVAL}, + { LTFS_XML_TOP_ATTR_FAIL, "I5013E", EINVAL}, + { LTFS_XML_WRONG_UUID, "I5014E", EINVAL}, + { LTFS_XML_WRONG_GEN, "I5015E", EINVAL}, + { LTFS_XML_WRONG_UTIME, "I5016E", EINVAL}, + { LTFS_XML_WRONG_LOC, "I5017E", EINVAL}, + { LTFS_XML_WRONG_LOC_PREV, "I5018E", EINVAL}, + { LTFS_XML_WRONG_PA, "I5019E", EINVAL}, + { LTFS_XML_WRONG_POLICY, "I5020E", EINVAL}, + { LTFS_XML_TOO_LONG_COMMENT, "I5021E", EINVAL}, + { LTFS_XML_WRONG_NEXT, "I5022E", EINVAL}, + { LTFS_XML_WRONG_RO_DIR, "I5023E", EINVAL}, + { LTFS_XML_WRONG_MTIME_DIR, "I5024E", EINVAL}, + { LTFS_XML_WRONG_CRTIME_DIR, "I5025E", EINVAL}, + { LTFS_XML_WRONG_ATIME_DIR, "I5026E", EINVAL}, + { LTFS_XML_WRONG_CTIME_DIR, "I5027E", EINVAL}, + { LTFS_XML_WRONG_BTIME_DIR, "I5028E", EINVAL}, + { LTFS_XML_XATTR_TYPE, "I5029E", EINVAL}, + { LTFS_XML_XATTR_SIZE, "I5030E", EINVAL}, + { LTFS_XML_WRONG_UID, "I5031E", EINVAL}, + { LTFS_XML_INVALID_UID, "I5032E", EINVAL}, + { LTFS_XML_WRONG_RO_F, "I5033E", EINVAL}, + { LTFS_XML_WRONG_MTIME_F, "I5034E", EINVAL}, + { LTFS_XML_WRONG_CRTIME_F, "I5035E", EINVAL}, + { LTFS_XML_WRONG_ATIME_F, "I5036E", EINVAL}, + { LTFS_XML_WRONG_CTIME_F, "I5037E", EINVAL}, + { LTFS_XML_WRONG_BTIME_F, "I5038E", EINVAL}, + { LTFS_XML_WRONG_SIZE, "I5039E", EINVAL}, + { LTFS_XML_WRONG_PART, "I5040E", EINVAL}, + { LTFS_XML_WRONG_START_BLK, "I5041E", EINVAL}, + { LTFS_XML_WRONG_OFFSET, "I5042E", EINVAL}, + { LTFS_XML_WRONG_BYTE_CNT, "I5043E", EINVAL}, + { LTFS_XML_WRONG_FILE_OFST, "I5044E", EINVAL}, + { LTFS_XML_EXT_OVERLAP, "I5045E", EINVAL}, + { LTFS_XML_EXT_TOO_LONG, "I5046E", EINVAL}, + { LTFS_XML_WRONG_FTIME_L, "I5047E", EINVAL}, + { LTFS_XML_WRONG_PART_MAP, "I5048E", EINVAL}, + { LTFS_XML_WRONG_BLOCKSIZE, "I5049E", EINVAL}, + { LTFS_XML_WRONG_COMP, "I5050E", EINVAL}, + { LTFS_BAD_INDEX_TYPE, "I5051E", EINVAL}, { EDEV_NO_SENSE, "D0000E", EIO}, { EDEV_OVERRUN, "D0002E", EIO}, { EDEV_UNDERRUN, "D0003E", ENODATA}, diff --git a/src/libltfs/arch/time_internal.h b/src/libltfs/arch/time_internal.h index 859c9450..71508d76 100644 --- a/src/libltfs/arch/time_internal.h +++ b/src/libltfs/arch/time_internal.h @@ -56,12 +56,19 @@ extern "C" { #ifdef mingw_PLATFORM #include "libltfs/arch/win/win_util.h" +#elif defined(__APPLE__) +#include +typedef time_t ltfs_time_t; +struct ltfs_timespec { + ltfs_time_t tv_sec; + long tv_nsec; +}; #else #include -typedef int64_t ltfs_time_t; +typedef int64_t ltfs_time_t; struct ltfs_timespec { - ltfs_time_t tv_sec; - long tv_nsec; + ltfs_time_t tv_sec; + long tv_nsec; }; #endif @@ -69,12 +76,12 @@ struct ltfs_timespec { #define TIMER_TYPE_OSX (0x0000000000000001) #define TIMER_TYPE_WINDOWS (0x0000000000000002) -#pragma pack(1) +#pragma pack(push, 1) struct timer_info { uint64_t type; uint64_t base; }; -#pragma pack(0) +#pragma pack(pop) #define LTFS_TIME_T_MAX (253402300799) /* 9999/12/31 23:59:59 UTC */ #define LTFS_TIME_T_MIN (-62167219200) /* 0000/01/01 00:00:00 UTC */ diff --git a/src/libltfs/fs.c b/src/libltfs/fs.c index 99c83547..38bd65a0 100644 --- a/src/libltfs/fs.c +++ b/src/libltfs/fs.c @@ -263,7 +263,7 @@ struct dentry * fs_allocate_dentry(struct dentry *parent, const char *name, cons d->platform_safe_name = NULL; } else if (name && !platform_safe_name) { d->name.name = strdup(name); - update_platform_safe_name(d, FALSE, idx); + update_platform_safe_name(d, false, idx); if (! d->name.name || ! d->platform_safe_name) { ltfsmsg(LTFS_ERR, 10001E, "fs_allocate_dentry: name"); if (d->name.name) @@ -650,6 +650,99 @@ void fs_split_path(char *path, char **filename, size_t len) } } +int fs_path_clean(const char *path, struct ltfs_index *idx) +{ + int ret = 0; + struct dentry *d = NULL, *parent = NULL; + char *tmp_path, *start, *end; + + CHECK_ARG_NULL(path, -LTFS_NULL_ARG); + CHECK_ARG_NULL(idx, -LTFS_NULL_ARG); + + tmp_path = strdup(path); + if (! tmp_path) { + ltfsmsg(LTFS_ERR, 10001E, "fs_path_clean: tmp_path"); + return -LTFS_NO_MEMORY; + } + + /* Get a reference count on the root dentry. Either it will be returned immediately, or it + * will be disposed later after the first path lookup. */ + acquirewrite_mrsw(&idx->root->meta_lock); + ++idx->root->numhandles; + releasewrite_mrsw(&idx->root->meta_lock); + + if (idx->root->dirty) + idx->root->dirty = false; + + /* Did the caller ask for the root dentry? */ + if (*path == '\0' || ! strcmp(path, "/")) { + goto out; + } + + start = tmp_path + 1; + end = tmp_path; + d = idx->root; + + while (end) { + end = strstr(start, "/"); + if (end) + *end = '\0'; + + acquireread_mrsw(&d->contents_lock); + + if (parent) + releaseread_mrsw(&parent->contents_lock); + parent = d; + d = NULL; + + ret = fs_directory_lookup(parent, start, &d); + if (ret < 0 || ! d) { + releaseread_mrsw(&parent->contents_lock); + fs_release_dentry(parent); + + if (ret == 0) + ret = -LTFS_NO_DENTRY; + goto out; + } + + /* Release the parent if we aren't keeping any locks on it. + * Since we know 'parent' has a child (d), it's guaranteed that parent is still linked + * into the file system tree. Therefore, fs_release_dentry is just a fancy way of + * decrementing the handle count... so do that. */ + acquirewrite_mrsw(&parent->meta_lock); + --parent->numhandles; + releasewrite_mrsw(&parent->meta_lock); + + if (d->dirty) + d->dirty = false; + + if (end) + start = end + 1; + } + + releaseread_mrsw(&parent->contents_lock); + +out: + free(tmp_path); + + return ret; +} + +int fs_dir_clean(struct dentry *d) +{ + struct name_list *list_ptr = NULL, *list_tmp = NULL; + CHECK_ARG_NULL(d, -LTFS_NULL_ARG); + + if (d->isdir) { + HASH_ITER(hh, d->child_list, list_ptr, list_tmp) { + fs_dir_clean(list_ptr->d); + } + } else + d->dirty = false; + + return 0; +} + /** * Dispose a dentry and all resources used by it, including the struct dentry itself. * @param dentry dentry to dispose. @@ -845,7 +938,7 @@ static struct name_list* fs_update_platform_safe_names_and_hash_table(struct den int fs_update_platform_safe_names(struct dentry* basedir, struct ltfs_index *idx, struct name_list *list) { struct name_list *list_ptr, *list_tmp; - int rc = 0; + int ret = 0; list = fs_update_platform_safe_names_and_hash_table(basedir, idx, list, false, false); // normal loop list = fs_update_platform_safe_names_and_hash_table(basedir, idx, list, true, false); // add dup name @@ -857,12 +950,12 @@ int fs_update_platform_safe_names(struct dentry* basedir, struct ltfs_index *idx HASH_DEL(list, list_ptr); free(list_ptr); } - rc = -1; + ret = -LTFS_SAFENAME_FAIL; } HASH_CLEAR(hh, list); - return rc; + return ret; } /** diff --git a/src/libltfs/fs.h b/src/libltfs/fs.h index 781c79d9..4fe904b0 100644 --- a/src/libltfs/fs.h +++ b/src/libltfs/fs.h @@ -146,4 +146,26 @@ int fs_path_lookup(const char *path, int flags, struct dentry **dentry, struct l */ void fs_split_path(char *path, char **filename, size_t len); +/** + * Cleanup d->dirty flag in the provided path + * + * @param path Path to search for, in UTF-8 NFC. The path should be checked + * for invalid characters by the caller. This function validates the length of each + * path component. If path points to an empty string, this function returns the + * root dentry. + * @param idx LTFS index to search. + * @return 0 on success (dentry found), -LTFS_NO_DENTRY if no dentry was found, -LTFS_NAMETOOLONG + * if any component of the path is too long, or another negative value if an internal + * (unexpected) error occurs. + */ +int fs_path_clean(const char *path, struct ltfs_index *idx); + +/** + * Cleanup d->dirty flag under the provided directory (dentry) recursively + * + * @param d dentry structure to clean. Expect a directory. + * @return 0 on success, otherwise on error + */ +int fs_dir_clean(struct dentry *d); + #endif /* __fs_helper_h */ diff --git a/src/libltfs/inc_journal.c b/src/libltfs/inc_journal.c new file mode 100644 index 00000000..9c4386e0 --- /dev/null +++ b/src/libltfs/inc_journal.c @@ -0,0 +1,707 @@ +/* +** +** OO_Copyright_BEGIN +** +** +** Copyright 2010, 2024 IBM Corp. All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. Neither the name of the copyright holder nor the names of its +** contributors may be used to endorse or promote products derived from +** this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +** AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +** IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +** ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +** LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +** CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +** SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +** INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +** CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +** ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +** POSSIBILITY OF SUCH DAMAGE. +** +** +** OO_Copyright_END +** +************************************************************************************* +** +** COMPONENT NAME: IBM Linear Tape File System +** +** FILE NAME: inc_journal.c +** +** DESCRIPTION: Journal handling for incremental index +** +** AUTHORS: Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** +** ORIGINAL LOGIC: David Pease +** pease@coati.com +** +************************************************************************************* +*/ + +#include "ltfs.h" +#include "fs.h" +#include "inc_journal.h" + +static int _allocate_jentry(struct jentry **e, char *path, struct dentry* d) +{ + struct jentry *ent = NULL; + + *e = NULL; + + ent = calloc(1, sizeof(struct jentry)); + if (!ent) { + ltfsmsg(LTFS_ERR, 10001E, "allocating a jentry"); + return -LTFS_NO_MEMORY; + } + + ent->id.full_path = path; + ent->id.uid = d->uid; + + *e = ent; + + return 0; +} + +static inline int _dispose_jentry(struct jentry *ent) +{ + if (ent) { + if (ent->id.full_path) + free(ent->id.full_path); + free(ent); + } + + return 0; +} + +/** + * Handle created object in the tree. + * + * Caller need to grab vol->index->dirty_lock outside of this function. + * + * @param ppath parent path name of the object + * @param d dentry created + * @param vol pointer to the LTFS volume + */ +int incj_create(char *ppath, struct dentry *d, struct ltfs_volume *vol) +{ + int ret = -1, len = -1; + char *full_path = NULL; + struct jentry *ent = NULL; + struct jcreated_entry *jdir = NULL, *jd = NULL; + + /* Skip journal modification because of an error */ + if (vol->journal_err) { + return 0; + } + + /* Skip if an ancestor is already created in this session */ + TAILQ_FOREACH(jd, &vol->created_dirs, list) { + char* cp = jd->path; + if (strstr(ppath, cp) == ppath) { + return 0; + } + } + + /* Create full path of created object and jentry */ + len = asprintf(&full_path, "%s/%s", ppath, d->name.name); + if (len < 0) { + ltfsmsg(LTFS_ERR, 10001E, "full path of a jentry"); + vol->journal_err = true; + return -LTFS_NO_MEMORY; + } + + ret = _allocate_jentry(&ent, full_path, d); + if (ret < 0) { + vol->journal_err = true; + free(full_path); + return ret; + } + + ent->reason = CREATE; + ent->dentry = d; + + HASH_ADD(hh, vol->journal, id, sizeof(struct jentry), ent); + + if (d->isdir) { + jdir = calloc(1, sizeof(struct jcreated_entry)); + if (!jdir) { + ltfsmsg(LTFS_ERR, 10001E, "allocating a jcreated_entry"); + return -LTFS_NO_MEMORY; + } + + /* NOTE: Use same pointer of ent because it's life is same */ + jdir->path = ent->id.full_path; + + TAILQ_INSERT_TAIL(&vol->created_dirs, jdir, list); + } + + return 0; +} + +/** + * Handle modified file in the tree. + * + * Caller need to grab vol->index->dirty_lock outside of this function. + * + * @param path path name of the object + * @param d dentry to be modified (for recording uid) + * @param vol pointer to the LTFS volume + */ +int incj_modify(char *path, struct dentry *d, struct ltfs_volume *vol) +{ + int ret = -1; + struct jentry *ent = NULL; + struct jcreated_entry *jd = NULL; + + /* Skip journal modification because of an error */ + if (vol->journal_err) { + return 0; + } + + /* Skip journal modification because it is already existed */ + HASH_FIND(hh, vol->journal, &d->uid, sizeof(struct jentry), ent); + if (ent) { + return 0; + } + + /* Skip if an ancestor is already created in this session */ + TAILQ_FOREACH(jd, &vol->created_dirs, list) { + char *cp = jd->path; + if (strstr(path, cp) == path) { + return 0; + } + } + + ret = _allocate_jentry(&ent, path, d); + if (ret < 0) { + vol->journal_err = true; + return ret; + } + + ent->reason = MODIFY; + ent->dentry = d; + + HASH_ADD(hh, vol->journal, id, sizeof(struct jentry), ent); + + return 0; +} + +/** + * Handle deleted file in the tree. + * + * Caller need to grab vol->index->dirty_lock outside of this function. + * + * @param path path name of the object + * @param d dentry to be removed (for recording uid) + * @param vol pointer to the LTFS volume + */ +int incj_rmfile(char *path, struct dentry *d, struct ltfs_volume *vol) +{ + int ret = -1; + char *full_path = NULL; + struct journal_id id; + struct jentry *ent = NULL; + struct jcreated_entry *jd = NULL; + + /* Skip journal modification because of an error */ + if (vol->journal_err) { + return 0; + } + + id.full_path = path; + id.uid = d->uid; + HASH_FIND(hh, vol->journal, &id, sizeof(struct jentry), ent); + if (ent) { + if (ent->reason == CREATE) { + /* + * Remove the entry because this file is newly created and deleted + * in one incremental index session + */ + HASH_DEL(vol->journal, ent); + return 0; + } else if (ent->reason == MODIFY) { + /* + * Override the existing entry to DELETE_FILE record. + */ + ent->reason = DELETE_FILE; + ent->dentry = NULL; + return 0; + } + } + + /* Skip if an ancestor is already created in this session */ + TAILQ_FOREACH(jd, &vol->created_dirs, list) { + char *cp = jd->path; + if (strstr(path, cp) == path) { + return 0; + } + } + + /* Create full path of deleted object and jentry */ + full_path = strdup(path); + if (!full_path) { + ltfsmsg(LTFS_ERR, 10001E, "duplicating a path for deleted file"); + vol->journal_err = true; + return -LTFS_NO_MEMORY; + } + + ret = _allocate_jentry(&ent, full_path, d); + if (ret < 0) { + vol->journal_err = true; + return ret; + } + + ent->reason = DELETE_FILE; + ent->name.percent_encode = d->name.percent_encode; + ent->name.name = strdup(d->name.name); + if (!ent->name.name) { + ltfsmsg(LTFS_ERR, 10001E, "duplicating a name of deleted file"); + vol->journal_err = true; + return -LTFS_NO_MEMORY; + } + + HASH_ADD(hh, vol->journal, id, sizeof(struct jentry), ent); + + return 0; +} + +/** + * Handle deleted directory in the tree. + * + * Caller need to grab vol->index->dirty_lock outside of this function. + * + * @param path path name of the object + * @param d dentry to be removed (for recording uid) + * @param vol pointer to the LTFS volume + */ +int incj_rmdir(char *path, struct dentry *d, struct ltfs_volume *vol) +{ + int ret = -1; + char *full_path = NULL; + struct jentry *ent = NULL, *je = NULL, *tmp = NULL; + struct jcreated_entry *jd = NULL, *dtmp = NULL; + + /* Skip journal modification because of an error */ + if (vol->journal_err) { + return 0; + } + + /* + * 1. Remove entry from created_dirs if created directory is removed in a same session + * 2. Skip if an ancestor is already created in this session + */ + TAILQ_FOREACH_SAFE(jd, &vol->created_dirs, list, dtmp) { + char *cp = jd->path; + if (strstr(path, cp) == path) { + if (!strcmp(path, cp)) { + TAILQ_REMOVE(&vol->created_dirs, jd, list); + /* + * NOTE: + * Do not free jd->path because it shall be freed into _dispose_jentry. + * jentry::id.full_path and jd->path points the same address + */ + } else { + return 0; + } + } + } + + /* Need to find existing children under this directory */ + HASH_ITER(hh, vol->journal, je, tmp) { + if (strstr(je->id.full_path, path) == je->id.full_path) { + HASH_DEL(vol->journal, je); + _dispose_jentry(je); + } + } + + /* Create full path of created object and jentry */ + full_path = strdup(path); + if (!full_path) { + ltfsmsg(LTFS_ERR, 10001E, "duplicating a path of deleted directory"); + vol->journal_err = true; + return -LTFS_NO_MEMORY; + } + + ret = _allocate_jentry(&ent, full_path, d); + if (ret < 0) { + vol->journal_err = true; + return ret; + } + + ent->reason = DELETE_DIRECTORY; + ent->name.percent_encode = d->name.percent_encode; + ent->name.name = strdup(d->name.name); + if (!ent->name.name) { + ltfsmsg(LTFS_ERR, 10001E, "duplicating a name of deleted directory"); + vol->journal_err = true; + return -LTFS_NO_MEMORY; + } + + HASH_ADD(hh, vol->journal, id, sizeof(struct jentry), ent); + + return 0; +} + +int incj_dispose_jentry(struct jentry *ent) +{ + return (_dispose_jentry(ent)); +} + +/** + * Clear all entries into the incremental journal + */ +int incj_clear(struct ltfs_volume *vol) +{ + struct jentry *je = NULL, *tmp = NULL; + struct jcreated_entry *jd = NULL, *dtmp = NULL; + + TAILQ_FOREACH_SAFE(jd, &vol->created_dirs, list, dtmp) { + TAILQ_REMOVE(&vol->created_dirs, jd, list); + } + + HASH_ITER(hh, vol->journal, je, tmp) { + HASH_DEL(vol->journal, je); + _dispose_jentry(je); + } + + return 0; +} + +/** + * Sort function for dump incremental journal + */ +static int _by_path(const struct jentry *a, const struct jentry *b) +{ + int ret = 0; + + ret = strcmp(a->id.full_path, b->id.full_path); + if (!ret) { + if (a->id.uid > b->id.uid) + ret = 1; + else + ret = -1; + } + + return ret; +} + +static inline int dig_path(char *p, struct ltfs_index *idx) +{ + int ret = 0; + char *path; + + path = strdup(p); + if (! path) { + ltfsmsg(LTFS_ERR, 10001E, "dig_path: path"); + return -LTFS_NO_MEMORY; + } + + ret = fs_path_clean(path, idx); + + free(path); + + return ret; +} + +void incj_sort(struct ltfs_volume *vol) +{ + HASH_SORT(vol->journal, _by_path); +} + +/** + * This is a function for debug. Print contents of the journal and the created + * directory list to stdout. + */ +void incj_dump(struct ltfs_volume *vol) +{ + char *prev_parent = NULL, *parent, *filename; + struct jcreated_entry *jd = NULL, *dtmp = NULL; + struct jentry *ent = NULL, *tmp = NULL; + char *reason[] = { "CREATE", "MODIFY", "DELFILE", "DELDIR" }; + + printf("===============================================================================\n"); + TAILQ_FOREACH_SAFE(jd, &vol->created_dirs, list, dtmp) { + printf("CREATED_DIR: %s\n", jd->path); + TAILQ_REMOVE(&vol->created_dirs, jd, list); + } + + printf("--------------------------------------------------------------------------------\n"); + incj_sort(vol); + HASH_ITER(hh, vol->journal, ent, tmp) { + printf("JOURNAL: %s, %llu, %s, ", ent->id.full_path, (unsigned long long)ent->id.uid, reason[ent->reason]); + if (!ent->dentry) + printf("no-dentry\n"); + else { + if (ent->dentry->isdir) { + printf("dir\n"); + if (ent->reason == CREATE) + fs_dir_clean(ent->dentry); + } else + printf("file\n"); + + parent = strdup(ent->id.full_path); + fs_split_path(parent, &filename, strlen(parent) + 1); + + if (prev_parent) { + if (strcmp(prev_parent, parent)) { + dig_path(parent, vol->index); + } + free(prev_parent); + } else { + dig_path(parent, vol->index); + } + prev_parent = parent; + ent->dentry->dirty = false; + } + + HASH_DEL(vol->journal, ent); + _dispose_jentry(ent); + } + + if (prev_parent) free(prev_parent); + + return; +} + +int incj_create_path_helper(const char *dpath, struct incj_path_helper **pm, struct ltfs_volume *vol) +{ + struct incj_path_helper *ipm; + char *wp = NULL, *tmp = NULL, *dname = NULL; + int ret = 0; + + *pm = NULL; + + ipm = calloc(1, sizeof(struct incj_path_helper)); + if (!ipm) { + ltfsmsg(LTFS_ERR, 10001E, "allocating a path helper"); + return -LTFS_NO_MEMORY; + } + + if (dpath[0] != '/') { + /* Provided path must be a absolute path */ + ltfsmsg(LTFS_ERR, 17302E, dpath); + free(ipm); + return -LTFS_INVALID_PATH; + } + + ipm->vol = vol; + + if (strcmp(dpath, "/") == 0) { + /* Provided path is the root, return good */ + *pm = ipm; + return 0; + } + + wp = strdup(dpath); + if (!wp) { + ltfsmsg(LTFS_ERR, 10001E, "duplicating a directory path for path helper"); + free(ipm); + return -LTFS_NO_MEMORY; + } + + for (dname = strtok_r(wp, "/", &tmp); dname != NULL; dname = strtok_r(NULL, "/", &tmp)) { + ret = incj_push_directory(dname, ipm); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17305E, ret); + free(wp); + incj_destroy_path_helper(ipm); + return ret; + } + } + + free(wp); + *pm = ipm; + + return 0; +} + +int incj_destroy_path_helper(struct incj_path_helper *pm) +{ + struct incj_path_element *cur, *next; + + cur = pm->head; + + while (cur) { + next = cur->next; + if (cur->d) + fs_release_dentry(cur->d); + if (cur->name) + free(cur->name); + free(cur); + cur = next; + } + + free(pm); + return 0; +} + +int incj_push_directory(char *name, struct incj_path_helper *pm) +{ + int ret = 0; + struct incj_path_element *ipelm = NULL, *cur_tail = NULL; + struct dentry *parent = NULL; + + ipelm = calloc(1, sizeof(struct incj_path_element)); + if (!ipelm) { + ltfsmsg(LTFS_ERR, 10001E, "allocating a path element on push"); + return -LTFS_NO_MEMORY; + } + + /* Set name field of new path element */ + ipelm->name = strdup(name); + if (!ipelm->name) { + ltfsmsg(LTFS_ERR, 10001E, "duplicating a path of pushing directory"); + incj_destroy_path_helper(pm); + return -LTFS_NO_MEMORY; + } + + /* Set dentry field of new path element */ + if (pm->elems) + parent = pm->tail->d; + else + parent = pm->vol->index->root; + + ret = fs_directory_lookup(parent, name, &ipelm->d); + if (ret) { + ltfsmsg(LTFS_ERR, 17306E, ret); + free(ipelm->name); + free(ipelm); + incj_destroy_path_helper(pm); + return -LTFS_INVALID_PATH; + } + + /* Modify path chain and # of elements */ + if (!pm->elems) { + pm->head = ipelm; + pm->tail = ipelm; + } else { + cur_tail = pm->tail; + cur_tail->next = ipelm; + ipelm->prev = cur_tail; + pm->tail = ipelm; + } + + pm->elems++; + + return 0; +} + +int incj_pop_directory(struct incj_path_helper *pm) +{ + struct incj_path_element *cur_tail = NULL, *new_tail = NULL; + + if (!pm->elems) { + /* Must have one or more elements */ + return -LTFS_UNEXPECTED_VALUE; + } + + cur_tail = pm->tail; + new_tail = cur_tail->prev; + + new_tail->next = NULL; + pm->tail = new_tail; + + pm->elems--; + if (!pm->elems) { + pm->head = NULL; + } + + if (cur_tail->d) + fs_release_dentry(cur_tail->d); + if (cur_tail->name) + free(cur_tail->name); + free(cur_tail); + + return 0; +} + +int incj_compare_path(struct incj_path_helper *now, struct incj_path_helper *next, + int *matches, int *pops, bool *perfect_match) +{ + int ret = 0, matched = 0; + struct incj_path_element *cur1 = NULL, *cur2 = NULL; + + *matches = 0; + *pops = 0; + *perfect_match = false; + + cur1 = now->head; + cur2 = next->head; + + if (!cur1 && !cur2) { + /* Both are root */ + *perfect_match = true; + return 0; + } + + while (cur1 && cur2) { + if (cur1->d != cur2->d) + break; + matched++; + cur1 = cur1->next; + cur2 = cur2->next; + } + + *matches = matched; + *pops = now->elems - *matches; + + if (!cur1 && !cur2) + *perfect_match = true; + + return ret; +} + +char* incj_get_path(struct incj_path_helper *pm) +{ + char *path = NULL, *path_old = NULL; + struct incj_path_element *cur = NULL; + int ret = 0; + + cur = pm->head; + + if (!cur) { + /* Root directory */ + ret = asprintf(&path, "/"); + if (ret < 0) { + /* memory allocation error */ + return NULL; + } + return path; + } + + while (cur) { + if (path) path_old = path; + ret = asprintf(&path, "%s/%s", path_old, cur->name); + if (ret < 0) { + /* memory allocation error */ + free(path_old); + return NULL; + } + free(path_old); + + cur++; + } + + if (path) path_old = path; + ret = asprintf(&path, "/%s", path_old); + if (ret < 0) { + /* memory allocation error */ + free(path_old); + return NULL; + } + free(path_old); + + return path; +} diff --git a/src/libltfs/inc_journal.h b/src/libltfs/inc_journal.h new file mode 100644 index 00000000..14784857 --- /dev/null +++ b/src/libltfs/inc_journal.h @@ -0,0 +1,137 @@ +/* +** +** OO_Copyright_BEGIN +** +** +** Copyright 2010, 2024 IBM Corp. All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. Neither the name of the copyright holder nor the names of its +** contributors may be used to endorse or promote products derived from +** this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +** AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +** IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +** ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +** LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +** CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +** SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +** INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +** CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +** ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +** POSSIBILITY OF SUCH DAMAGE. +** +** +** OO_Copyright_END +** +************************************************************************************* +** +** COMPONENT NAME: IBM Linear Tape File System +** +** FILE NAME: inc_journal.h +** +** DESCRIPTION: Journal handling for incremental index +** +** AUTHORS: Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** +** ORIGINAL LOGIC: David Pease +** pease@coati.com +** +************************************************************************************* +*/ + +#include "queue.h" + +#ifndef __inc_journal_h__ +#define __inc_journal_h__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Enumeration of reasons for an entry + */ +enum journal_reason { + CREATE = 0, /**< Newly created, need to be 0 for debug */ + MODIFY, /**< Modified */ + DELETE_FILE, /**< File is deleted */ + DELETE_DIRECTORY, /**< Directory is deleted */ +}; + +/** + * Identifier of journal entry for handling multiple changes in one session + */ +struct journal_id { + char *full_path; /**< Full path name of the target */ + uint64_t uid; /**< i-node number of the target */ +}; + +/** + * Journal entry + */ +struct jentry { + struct journal_id id; /**< ID of the journal entry (key of the hash table) */ + enum journal_reason reason; /**< Reason of the entry */ + struct dentry *dentry; /**< Target dentry if required */ + struct ltfs_name name; /**< Name of entry for delete */ + UT_hash_handle hh; +}; + +/** + * Created directory list + */ +struct jcreated_entry { + TAILQ_ENTRY(jcreated_entry) list; /**< Pointers for linked list of requests */ + char *path; +}; + +/** + * + */ +struct incj_path_element { + struct incj_path_element *prev; + struct incj_path_element *next; + char* name; + struct dentry *d; +}; + +struct incj_path_helper { + struct incj_path_element *head; + struct incj_path_element *tail; + struct ltfs_volume *vol; + unsigned int elems; +}; + +int incj_create(char *ppath, struct dentry *d, struct ltfs_volume *vol); +int incj_modify(char *path, struct dentry *d, struct ltfs_volume *vol); +int incj_rmfile(char *path, struct dentry *d, struct ltfs_volume *vol); +int incj_rmdir(char *path, struct dentry *d, struct ltfs_volume *vol); +int incj_dispose_jentry(struct jentry *ent); +int incj_clear(struct ltfs_volume *vol); +void incj_sort(struct ltfs_volume *vol); +void incj_dump(struct ltfs_volume *vol); + +int incj_create_path_helper(const char *path, struct incj_path_helper **pm, struct ltfs_volume *vol); +int incj_destroy_path_helper(struct incj_path_helper *pm); +int incj_push_directory(char *name, struct incj_path_helper *pm); +int incj_pop_directory(struct incj_path_helper *pm); +int incj_compare_path(struct incj_path_helper *p1, struct incj_path_helper *p2, + int *matches, int *pops, bool *perfect_match); +char* incj_get_path(struct incj_path_helper *pm); + +#ifdef __cplusplus +} +#endif + +#endif /* __inc_journal_h__ */ diff --git a/src/libltfs/index_criteria.c b/src/libltfs/index_criteria.c index c6c9ecff..88819b16 100644 --- a/src/libltfs/index_criteria.c +++ b/src/libltfs/index_criteria.c @@ -200,9 +200,17 @@ bool index_criteria_find_option(const char *str, const char *substr, int index_criteria_parse_size(const char *criteria, size_t len, struct index_criteria *ic) { int ret = 0, multiplier = 1; + size_t sizelen = 0; char rule[len+1], last, *ptr; - snprintf(rule, len-strlen("size="), "%s", criteria + strlen("size=")); + sizelen = strlen("size="); + if (len <= sizelen) { + ltfsmsg(LTFS_ERR, 11143E, len); + return -LTFS_POLICY_INVALID; + } + + memset(rule, 0, sizeof(rule)); + snprintf(rule, len - sizelen, "%s", criteria + sizelen); for (ptr=&rule[0]; *ptr; ptr++) { if (isalpha(*ptr) && *(ptr+1) && isalpha(*(ptr+1))) { diff --git a/src/libltfs/ltfs.c b/src/libltfs/ltfs.c index fa6a358c..7b74b37b 100644 --- a/src/libltfs/ltfs.c +++ b/src/libltfs/ltfs.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -70,6 +70,7 @@ #include "xattr.h" #include "xml_libltfs.h" #include "label.h" +#include "inc_journal.h" #include "arch/version.h" #include "arch/filename_handling.h" #include "libltfs/arch/errormap.h" @@ -110,6 +111,17 @@ const char *ltfs_format_version() return LTFS_INDEX_VERSION_STR; } +static inline char* _get_barcode(struct ltfs_volume *vol) +{ + char *barcode = NULL; + + if (vol->label->barcode[0] != ' ') + barcode = vol->label->barcode; + else + barcode = LTFS_NO_BARCODE; + + return barcode; +} /** * Initialize the LTFS functions, currently the XML parser and the logging component. @@ -362,6 +374,9 @@ int ltfs_volume_alloc(const char *execname, struct ltfs_volume **volume) } } + newvol->journal = NULL; + TAILQ_INIT(&newvol->created_dirs); + *volume = newvol; return 0; @@ -404,8 +419,10 @@ void _ltfs_volume_free(bool force, struct ltfs_volume **volume) free((*volume)->mountpoint); if ((*volume)->t_attr) free((*volume)->t_attr); - if ((*volume)->index_cache_path) - free((*volume)->index_cache_path); + if ((*volume)->index_cache_path_w) + free((*volume)->index_cache_path_w); + if ((*volume)->index_cache_path_r) + free((*volume)->index_cache_path_r); destroy_mrsw(&(*volume)->lock); ltfs_thread_mutex_destroy(&(*volume)->reval_lock); ltfs_thread_cond_destroy(&(*volume)->reval_cond); @@ -535,6 +552,13 @@ int ltfs_test_unit_ready(struct ltfs_volume *vol) ret = ltfs_get_volume_lock(false, vol); if (ret < 0) return ret; + + if (vol->mount_type == MOUNT_ROLLBACK_META) { + /* Return good when volume is meta-data mount mode */ + releaseread_mrsw(&vol->lock); + return 0; + } + ret = tape_device_lock(vol->device); if (ret == -LTFS_DEVICE_FENCED) { ret = ltfs_wait_revalidation(vol); @@ -631,6 +655,13 @@ int ltfs_capacity_data(struct device_capacity *cap, struct ltfs_volume *vol) ret = ltfs_get_volume_lock(false, vol); if (ret < 0) return ret; + + if (vol->mount_type == MOUNT_ROLLBACK_META) { + /* Return good when volume is meta-data mount mode */ + releaseread_mrsw(&vol->lock); + return 0; + } + ret = ltfs_capacity_data_unlocked(cap, vol); if (ret == -LTFS_DEVICE_FENCED) { ret = ltfs_wait_revalidation(vol); @@ -1445,6 +1476,51 @@ int ltfs_start_mount(bool trial, struct ltfs_volume *vol) return 0; } +static inline int _ltfs_search_index_wp(bool recover_symlink, bool can_skip_ip, + struct tc_position *seekpos, struct ltfs_volume *vol) +{ + int ret = 0; + tape_block_t end_pos, index_end_pos; + bool fm_after, blocks_after; + + ltfsmsg(LTFS_INFO, 17284I, "IP"); + ret = ltfs_seek_index(vol->label->partid_ip, &end_pos, &index_end_pos, &fm_after, + &blocks_after, recover_symlink, vol); + if (ret) { + if (can_skip_ip) { + ltfsmsg(LTFS_INFO, 17289I); + vol->ip_coh.count = 0; + vol->ip_coh.set_id = 0; + } else { + ltfsmsg(LTFS_ERR, 17285E, "IP", ret); + return -LTFS_INDEX_INVALID; + } + } + + ltfsmsg(LTFS_INFO, 17284I, "DP"); + ret = ltfs_seek_index(vol->label->partid_dp, &end_pos, &index_end_pos, &fm_after, + &blocks_after, recover_symlink, vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17285E, "DP", ret); + return -LTFS_INDEX_INVALID; + } + + /* Use the latest index on the tape */ + ltfsmsg(LTFS_INFO, 17288I, + (unsigned long long)vol->ip_coh.count, (unsigned long long)vol->ip_coh.set_id, + (unsigned long long)vol->dp_coh.count, (unsigned long long)vol->dp_coh.set_id); + + if (vol->ip_coh.count > vol->dp_coh.count) { + seekpos->partition = ltfs_part_id2num(vol->label->partid_ip, vol); + seekpos->block = vol->ip_coh.set_id; + } else { + seekpos->partition = ltfs_part_id2num(vol->label->partid_dp, vol); + seekpos->block = vol->dp_coh.set_id; + } + + return 0; +} + /** * Read LTFS data structures from a tape, checking for consistency (and restoring it * if possible). This function doesn't bother locking vol->index, as it must complete before any @@ -1468,8 +1544,9 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec /* TODO: is_worm_recovery_mount should be set by user via option */ int vollock = UNLOCKED_MAM; char *vl_print = NULL; + char *barcode = NULL; - ltfsmsg(LTFS_INFO, 11005I); + ltfsmsg(LTFS_INFO, 11005I, "device"); CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); @@ -1585,8 +1662,27 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec */ ltfsmsg(LTFS_INFO, 17264I, "DP", vl_print); } - seekpos.partition = ltfs_part_id2num(vol->label->partid_dp, vol); - seekpos.block = vol->dp_coh.set_id; + + if (volume_change_ref != vol->dp_coh.volume_change_ref) { + /* + * Cannot trust the index info on MAM, search the last indexes + * This would happen when the drive returns an error against acquiring the VCR + * while write error handling. + */ + ltfsmsg(LTFS_INFO, 17283I, + (unsigned long long)vol->dp_coh.volume_change_ref, + (unsigned long long)volume_change_ref); + + /* Index of IP could be corrupted. So set skip flag */ + ret = _ltfs_search_index_wp(recover_symlink, true, &seekpos, vol); + if (ret < 0) + goto out_unlock; + + } else { + ltfsmsg(LTFS_INFO, 17286I, "DP", (unsigned long long)volume_change_ref); + seekpos.partition = ltfs_part_id2num(vol->label->partid_dp, vol); + seekpos.block = vol->dp_coh.set_id; + } } else { if (vollock != PWE_MAM_DP && vollock != PWE_MAM) { /* @@ -1595,13 +1691,36 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec * so this condition wouldn't happen logically. */ ltfsmsg(LTFS_INFO, 17264I, "IP", vl_print); - tape_takedump_drive(vol->device, false); } - seekpos.partition = ltfs_part_id2num(vol->label->partid_ip, vol); - seekpos.block = vol->ip_coh.set_id; - read_ip = true; + + if (volume_change_ref != vol->ip_coh.volume_change_ref) { + /* + * Cannot trust the index info on MAM, search the last indexes + * This would happen when the drive returns an error against acquiring the VCR + * while write error handling. + */ + ltfsmsg(LTFS_INFO, 17283I, + (unsigned long long)vol->dp_coh.volume_change_ref, + (unsigned long long)volume_change_ref); + + ret = _ltfs_search_index_wp(recover_symlink, false, &seekpos, vol); + if (ret < 0) + goto out_unlock; + + } else { + ltfsmsg(LTFS_INFO, 17286I, "IP", (unsigned long long)volume_change_ref); + seekpos.partition = ltfs_part_id2num(vol->label->partid_ip, vol); + seekpos.block = vol->ip_coh.set_id; + } } + if (vol->label->part_num2id[seekpos.partition] == vol->label->partid_ip) + read_ip = true; + + ltfsmsg(LTFS_INFO, 17287I, + vol->label->part_num2id[seekpos.partition], + (unsigned long long)seekpos.block); + ret = tape_seek(vol->device, &seekpos); if (ret == -EDEV_EOD_DETECTED) { INTERRUPTED_GOTO(ret, out_unlock); @@ -1620,15 +1739,14 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec goto out_unlock; } else { INTERRUPTED_GOTO(ret, out_unlock); - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, false, vol); if (ret < 0) { if (read_ip) ltfsmsg(LTFS_ERR, 11024E); /* read IP Index failed */ else ltfsmsg(LTFS_ERR, 11021E); /* read DP Index failed */ goto out_unlock; - } - else { + } else { ltfsmsg(LTFS_DEBUG, 11025D); /* volume is consistent */ vol->mount_type = MOUNT_ERR_TAPE; } @@ -1654,14 +1772,15 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec goto out_unlock; } else { INTERRUPTED_GOTO(ret, out_unlock); - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, false, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 11021E); /* read DP Index failed */ goto out_unlock; } INTERRUPTED_GOTO(ret, out_unlock); ltfsmsg(LTFS_INFO, 11022I); - ret = ltfs_write_index(vol->label->partid_ip, SYNC_RECOVERY, vol); + ltfs_set_commit_message_reason(SYNC_RECOVERY, vol); + ret = ltfs_write_index(vol->label->partid_ip, SYNC_RECOVERY, LTFS_FULL_INDEX, vol); if (ret < 0) goto out_unlock; } @@ -1683,7 +1802,7 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec goto out_unlock; } else { INTERRUPTED_GOTO(ret, out_unlock); - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, false, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 11024E); /* read IP Index failed */ goto out_unlock; @@ -1710,17 +1829,17 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec INTERRUPTED_GOTO(ret, out_unlock); if(gen != 0 && gen != vol->index->generation) { if(is_worm_recovery_mount){ - ret = ltfs_traverse_index_no_eod(vol, ltfs_ip_id(vol), gen, NULL, NULL, NULL); + ret = ltfs_traverse_index_no_eod(vol, ltfs_ip_id(vol), gen, false, NULL, NULL, NULL); if(ret < 0) - ret = ltfs_traverse_index_no_eod(vol, ltfs_dp_id(vol), gen, NULL, NULL, NULL); + ret = ltfs_traverse_index_no_eod(vol, ltfs_dp_id(vol), gen, false, NULL, NULL, NULL); } else if(vol->traverse_mode == TRAVERSE_FORWARD){ - ret = ltfs_traverse_index_forward(vol, ltfs_ip_id(vol), gen, NULL, NULL, NULL); + ret = ltfs_traverse_index_forward(vol, ltfs_ip_id(vol), gen, false, NULL, NULL, NULL); if(ret < 0) - ret = ltfs_traverse_index_forward(vol, ltfs_dp_id(vol), gen, NULL, NULL, NULL); + ret = ltfs_traverse_index_forward(vol, ltfs_dp_id(vol), gen, false, NULL, NULL, NULL); } else { - ret = ltfs_traverse_index_backward(vol, ltfs_ip_id(vol), gen, NULL, NULL, NULL); + ret = ltfs_traverse_index_backward(vol, ltfs_ip_id(vol), gen, false, NULL, NULL, NULL); if(ret < 0) - ret = ltfs_traverse_index_backward(vol, ltfs_dp_id(vol), gen, NULL, NULL, NULL); + ret = ltfs_traverse_index_backward(vol, ltfs_dp_id(vol), gen, false, NULL, NULL, NULL); } if (ret < 0) { ltfsmsg(LTFS_ERR, 17079E, gen); @@ -1746,13 +1865,6 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec if (vol->index->uid_number == 0) ltfsmsg(LTFS_WARN, 11307W, vol->label->vol_uuid); - /* Clear the commit message so it doesn't carry over from the previous session */ - /* TODO: is this the right place to clear the commit message? */ - if (vol->index->commit_message) { - free(vol->index->commit_message); - vol->index->commit_message = NULL; - } - /* If we reach this point, both partitions end in an index file. */ vol->ip_index_file_end = true; vol->dp_index_file_end = true; @@ -1778,6 +1890,17 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec vol->lock_status = vol->index->vollock; } + barcode = _get_barcode(vol); + + ltfsmsg(LTFS_INFO, 11031I, + barcode, + (unsigned long long)vol->index->generation, + vol->index->selfptr.partition, + (unsigned long long)vol->index->selfptr.block, + vol->index->backptr.partition, + (unsigned long long)vol->index->backptr.block, + tape_get_serialnumber(vol->device)); + out_unlock: if (index && vol->index) ltfs_index_free(&index); @@ -1790,6 +1913,62 @@ int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool rec return ret; } +/** + * Mount LTFS volume from provided index file + * Volume shall be mounted as read-only to avoid illegal data update. + * + * @param filename file name of the index + * @param label_check check label on tape (for R/O mount with data access) on true + * @param vol the volume to load + * @return 0 on success or a negative value on error. + */ +int ltfs_mount_indexfile(char* filename, bool label_check, struct ltfs_volume *vol) +{ + int ret = 0; + + ltfsmsg(LTFS_INFO, 11005I, filename); + + CHECK_ARG_NULL(filename, -LTFS_NULL_ARG); + CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); + + if (label_check) { + /* load tape, read indexes, set compression */ + ret = ltfs_start_mount(false, vol); + if (ret < 0) { + /* ltfs_start_mount() generated an appropriate error message */ + goto out_unlock; + } + ltfsmsg(LTFS_DEBUG, 11013D); /* partition labels are valid */ + vol->mount_type = MOUNT_ROLLBACK; + } else { + /* Assume 512KB block*/ + vol->label->blocksize = 512 * KB; + vol->mount_type = MOUNT_ROLLBACK_META; + } + + vol->first_locate.tv_sec = 0; + vol->first_locate.tv_nsec = 0; + + ret = ltfs_read_indexfile(filename, false, vol); + + if (label_check) { + if (strcmp(vol->index->vol_uuid, vol->label->vol_uuid)) { + /* + * Volume UUID in label and it on index is not matched. + * Actual UUID is not printed to avoid illegal modification by hand. + */ + ltfsmsg(LTFS_ERR, 17293E); + ret = -LTFS_LABEL_MISMATCH; + } + } + +out_unlock: + if (ret < 0 && vol->index) + ltfs_index_free(&vol->index); + + return ret; +} + /** * Load cartridge attribute varues from CM * @param vol the volume to get attribute @@ -1837,10 +2016,7 @@ void ltfs_set_index_dirty(bool locking, bool atime, struct ltfs_index *idx) ltfs_mutex_unlock(&idx->dirty_lock); if (!was_dirty && idx->dirty) { - if (idx->root->vol->label->barcode[0] != ' ') - ltfsmsg(LTFS_INFO, 11337I, true, idx->root->vol->label->barcode, idx->root->vol); - else - ltfsmsg(LTFS_INFO, 11337I, true, LTFS_NO_BARCODE, idx->root->vol); + ltfsmsg(LTFS_INFO, 11337I, true, _get_barcode(idx->root->vol), idx->root->vol); } } } @@ -1866,10 +2042,7 @@ void ltfs_unset_index_dirty(bool update_version, struct ltfs_index *idx) ltfs_mutex_unlock(&idx->dirty_lock); if (was_dirty && !idx->dirty) { - if (idx->root->vol->label->barcode[0] != ' ') - ltfsmsg(LTFS_INFO, 11337I, false, idx->root->vol->label->barcode, idx->root->vol); - else - ltfsmsg(LTFS_INFO, 11337I, false, LTFS_NO_BARCODE, idx->root->vol); + ltfsmsg(LTFS_INFO, 11337I, false, _get_barcode(idx->root->vol), idx->root->vol); } } } @@ -1886,7 +2059,6 @@ int ltfs_unmount(char *reason, struct ltfs_volume *vol) int ret; cartridge_health_info h; int vollock = UNLOCKED_MAM; - char *skip_reason = NULL; char *mount_type = NULL; char *mam_lock = NULL; @@ -1894,13 +2066,14 @@ int ltfs_unmount(char *reason, struct ltfs_volume *vol) start: ret = ltfs_get_volume_lock(true, vol); - if (!ret) { + if (!ret && vol->mount_type != MOUNT_ROLLBACK_META) { + ret = tape_get_cart_volume_lock_status(vol->device, &vollock); if (vol->mount_type == MOUNT_NORMAL && (ltfs_is_dirty(vol) || vol->index->selfptr.partition != ltfs_ip_id(vol)) && (vollock != PWE_MAM_IP && vollock != PWE_MAM_BOTH)) { - ret = ltfs_write_index(ltfs_ip_id(vol), reason, vol); + ret = ltfs_write_index(ltfs_ip_id(vol), reason, LTFS_FULL_INDEX, vol); if (NEED_REVAL(ret)) { ret = ltfs_revalidate(true, vol); if (ret == 0) { @@ -2173,7 +2346,7 @@ void ltfs_set_work_dir(const char *dir, struct ltfs_volume *vol) /** * Configure EOD (end of data) checking for a volume. This should be done before - * calling ltfs_mount. + * calling ltfs_mount * The EOD check is enabled by default. */ void ltfs_set_eod_check(bool use, struct ltfs_volume *vol) @@ -2262,13 +2435,15 @@ size_t ltfs_max_cache_size(struct ltfs_volume *vol) * when the cartridge is known to be in a sane state. * The caller must hold vol->lock for write if thread safety is required. * @param partition partition in which the schema should be written to + * @param reason the reason to write down an index + * @param type type of index to write (shall be LTFS_FULL_INDEX or LTFS_INCREMENTAL_INDEX) * @param vol LTFS volume * @return 0 on success or a negative value on error */ -int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) +int ltfs_write_index(char partition, char *reason, enum ltfs_index_type type, struct ltfs_volume *vol) { int ret, ret_mam; - struct tape_offset old_selfptr, old_backptr; + struct tape_offset old_selfptr, old_backptr, old_selfptr_inc, old_backptr_inc; struct ltfs_timespec modtime_old = { .tv_sec = 0, .tv_nsec = 0 }; bool generation_inc = false; struct tc_position physical_selfptr; @@ -2286,10 +2461,32 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) return ret; } - if (vol->label->barcode[0] == ' ' || vol->label->barcode == NULL) - bc_print = LTFS_NO_BARCODE; - else - bc_print = vol->label->barcode; + if (type == LTFS_INDEX_AUTO) { + if (vol->index->full_index_interval) { + if (vol->index->full_index_to_go) + type = LTFS_INCREMENTAL_INDEX; + else + type = LTFS_FULL_INDEX; + } else { + type = LTFS_FULL_INDEX; + } + } + + switch (type) { + case LTFS_FULL_INDEX: + if (vol->index->full_index_interval) + vol->index->full_index_to_go = vol->index->full_index_interval; + break; + case LTFS_INCREMENTAL_INDEX: + if (vol->index->full_index_interval && vol->index->full_index_to_go) + vol->index->full_index_to_go--; + break; + default: + /* TODO: Unexpected index type error */ + break; + } + + bc_print = _get_barcode(vol); if (write_perm) { ltfsmsg(LTFS_INFO, 11343I, bc_print); @@ -2323,12 +2520,15 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) (vol->ip_index_file_end && vol->index->selfptr.partition == ltfs_ip_id(vol)))) { /* Surpress on-disk index cache write on the recursive call */ - cache_path_save = vol->index_cache_path; - vol->index_cache_path = NULL; - ret = ltfs_write_index(ltfs_dp_id(vol), reason, vol); + cache_path_save = vol->index_cache_path_w; + vol->index_cache_path_w = NULL; + + type = LTFS_FULL_INDEX; + incj_clear(vol); /* Clear incremental journal data */ + ret = ltfs_write_index(ltfs_dp_id(vol), reason, type, vol); /* Restore cache path to handle on-disk index cache */ - vol->index_cache_path = cache_path_save; + vol->index_cache_path_w = cache_path_save; cache_path_save = NULL; if (NEED_REVAL(ret)) @@ -2357,16 +2557,17 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) write_perm = true; reason = SYNC_WRITE_PERM; + ltfs_set_commit_message_reason(SYNC_WRITE_PERM, vol); } /* ignore return value: we want to keep trying even if, e.g., the DP fills up */ } /* update index generation */ if (ltfs_is_dirty(vol)) { - modtime_old = vol->index->mod_time; generation_inc = true; - get_current_timespec(&vol->index->mod_time); ++vol->index->generation; + modtime_old = vol->index->mod_time; + get_current_timespec(&vol->index->mod_time); } /* locate to append position */ @@ -2381,38 +2582,61 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) } /* update back pointer */ - old_backptr = vol->index->backptr; - if (vol->index->selfptr.partition == ltfs_dp_id(vol)) - memcpy(&vol->index->backptr, &vol->index->selfptr, sizeof(struct tape_offset)); + if (type == LTFS_FULL_INDEX) { + old_backptr = vol->index->backptr; + if (vol->index->selfptr.partition == ltfs_dp_id(vol)) + memcpy(&vol->index->backptr, &vol->index->selfptr, sizeof(struct tape_offset)); + } else { + old_backptr_inc = vol->index->backptr_inc; + memcpy(&vol->index->backptr_inc, &vol->index->selfptr_inc, sizeof(struct tape_offset)); + } /* update self pointer */ ret = tape_get_position(vol->device, &physical_selfptr); if (ret < 0) { ltfsmsg(LTFS_ERR, 11081E, ret); - if (generation_inc) { - vol->index->mod_time = modtime_old; - --vol->index->generation; + if (type == LTFS_FULL_INDEX) { + if (generation_inc) { + vol->index->mod_time = modtime_old; + --vol->index->generation; + } + vol->index->backptr = old_backptr; + } else { + vol->index->backptr_inc = old_backptr_inc; } - vol->index->backptr = old_backptr; goto out_write_perm; } - old_selfptr = vol->index->selfptr; - vol->index->selfptr.partition = partition; - vol->index->selfptr.partition = vol->label->part_num2id[physical_selfptr.partition]; - vol->index->selfptr.block = physical_selfptr.block; - ++vol->index->selfptr.block; /* point to first data block, not preceding filemark */ + + if (type == LTFS_FULL_INDEX) { + old_selfptr = vol->index->selfptr; + vol->index->selfptr.partition = partition; + vol->index->selfptr.partition = vol->label->part_num2id[physical_selfptr.partition]; + vol->index->selfptr.block = physical_selfptr.block; + ++vol->index->selfptr.block; /* point to first data block, not preceding filemark */ + } else { + old_selfptr_inc = vol->index->selfptr_inc; + vol->index->selfptr_inc.partition = partition; + vol->index->selfptr_inc.partition = vol->label->part_num2id[physical_selfptr.partition]; + vol->index->selfptr_inc.block = physical_selfptr.block; + ++vol->index->selfptr_inc.block; /* point to first data block, not preceding filemark */ + } /* Write the Index. */ if ((partition == ltfs_ip_id(vol)) && !vol->ip_index_file_end) { ret = tape_write_filemark(vol->device, 0, true, true, false); // Flush data before writing FM if (ret < 0) { ltfsmsg(LTFS_ERR, 11326E, ret); - if (generation_inc) { - vol->index->mod_time = modtime_old; - --vol->index->generation; + if (type == LTFS_FULL_INDEX) { + if (generation_inc) { + vol->index->mod_time = modtime_old; + --vol->index->generation; + } + vol->index->backptr = old_backptr; + vol->index->selfptr = old_selfptr; + } else { + vol->index->backptr_inc = old_backptr_inc; + vol->index->selfptr_inc = old_selfptr_inc; } - vol->index->backptr = old_backptr; - vol->index->selfptr = old_selfptr; if (IS_WRITE_PERM(-ret)) update_vollock = true; @@ -2427,12 +2651,18 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) ret = tape_write_filemark(vol->device, 1, true, true, true); // immediate WFM if (ret < 0) { ltfsmsg(LTFS_ERR, 11082E, ret); - if (generation_inc) { - vol->index->mod_time = modtime_old; - --vol->index->generation; + + if (type == LTFS_FULL_INDEX) { + if (generation_inc) { + vol->index->mod_time = modtime_old; + --vol->index->generation; + } + vol->index->backptr = old_backptr; + vol->index->selfptr = old_selfptr; + } else { + vol->index->backptr_inc = old_backptr_inc; + vol->index->selfptr_inc = old_selfptr_inc; } - vol->index->backptr = old_backptr; - vol->index->selfptr = old_selfptr; if (IS_WRITE_PERM(-ret)) update_vollock = true; @@ -2441,15 +2671,21 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) } /* Actually write index to tape and disk if vol->index_cache_path is existed */ - ret = xml_schema_to_tape(reason, vol); + ret = xml_schema_to_tape(reason, type, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 11083E, ret); - if (generation_inc) { - vol->index->mod_time = modtime_old; - --vol->index->generation; + + if (type == LTFS_FULL_INDEX) { + if (generation_inc) { + vol->index->mod_time = modtime_old; + --vol->index->generation; + } + vol->index->backptr = old_backptr; + vol->index->selfptr = old_selfptr; + } else { + vol->index->backptr_inc = old_backptr_inc; + vol->index->selfptr_inc = old_selfptr_inc; } - vol->index->backptr = old_backptr; - vol->index->selfptr = old_selfptr; if (IS_WRITE_PERM(-ret)) update_vollock = true; @@ -2457,33 +2693,60 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) goto out_write_perm; } - /* Update MAM parameters. */ - if (partition == ltfs_ip_id(vol)) - vol->ip_index_file_end = true; - else /* partition == ltfs_dp_id(vol) */ - vol->dp_index_file_end = true; - - /* The MAM may be inaccessible, or it may not be available on this medium. Either way, - * ignore failures when updating MAM parameters. */ - ltfs_update_cart_coherency(vol); + if (type == LTFS_FULL_INDEX) { + /* Update MAM parameters. */ + if (partition == ltfs_ip_id(vol)) + vol->ip_index_file_end = true; + else /* partition == ltfs_dp_id(vol) */ + vol->dp_index_file_end = true; + + /* The MAM may be inaccessible, or it may not be available on this medium. Either way, + * ignore failures when updating MAM parameters. */ + ltfs_update_cart_coherency(vol); + + ltfsmsg(LTFS_INFO, 17236I, + bc_print, + (unsigned long long)vol->index->generation, + vol->index->selfptr.partition, + (unsigned long long)vol->index->selfptr.block, + tape_get_serialnumber(vol->device)); + + /* update append position */ + if (partition == ltfs_ip_id(vol)) { + tape_set_ip_append_position(vol->device, ltfs_part_id2num(partition, vol), + vol->index->selfptr.block - 1); + } - ltfsmsg(LTFS_INFO, 17236I, bc_print, partition, tape_get_serialnumber(vol->device)); + if (dcache_initialized(vol)) { + dcache_set_dirty(false, vol); + if (generation_inc) { + dcache_set_generation(vol->index->generation, vol); + } + } - /* update append position */ - if (partition == ltfs_ip_id(vol)) { - tape_set_ip_append_position(vol->device, ltfs_part_id2num(partition, vol), - vol->index->selfptr.block - 1); - } + /* Clear incremental index metrix */ + vol->index->backptr_inc.partition = 0; + vol->index->backptr_inc.block = 0; + vol->index->selfptr_inc.partition = 0; + vol->index->selfptr_inc.block = 0; - if (dcache_initialized(vol)) { - dcache_set_dirty(false, vol); - if (generation_inc) { - dcache_set_generation(vol->index->generation, vol); - } + incj_clear(vol); /* Clear incremental journal data */ + ltfs_unset_index_dirty(true, vol->index); + } else { + ltfsmsg(LTFS_INFO, 17300I, + bc_print, + (unsigned long long)vol->index->generation, + vol->index->selfptr_inc.partition, + (unsigned long long)vol->index->selfptr_inc.block, + tape_get_serialnumber(vol->device)); + ltfsmsg(LTFS_INFO, 17301I, + (unsigned long long)vol->index->generation, + vol->index->backptr.partition, + (unsigned long long)vol->index->backptr.block, + vol->index->backptr_inc.partition, + (unsigned long long)vol->index->backptr_inc.block); } - ltfs_unset_index_dirty(true, vol->index); - out_write_perm: if (write_perm) { /* Restore device->write_error flag that is disabled to handle a write perm on DP */ @@ -2516,11 +2779,11 @@ int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol) * Write down the state of the current LTFS file system to a XML file on disk. * Acquires a write lock on vol->index->root->lock. * @param work_dir LTFS work directory. - * @param need_gen include generation number to file name + * @param id partition ID or 0 * @param vol LTFS volume * @return 0 on success or a negative value on error */ -int ltfs_save_index_to_disk(const char *work_dir, char * reason, bool need_gen, struct ltfs_volume *vol) +int ltfs_save_index_to_disk(const char *work_dir, char * reason, char id, struct ltfs_volume *vol) { char *path; int ret; @@ -2532,13 +2795,13 @@ int ltfs_save_index_to_disk(const char *work_dir, char * reason, bool need_gen, /* Write the schema to a file on disk */ ltfsmsg(LTFS_DEBUG, 17182D, vol->label->vol_uuid, vol->label->barcode); - if (need_gen) { - if (strcmp(vol->label->barcode, " ")) - ret = asprintf(&path, "%s/%s-%d.schema", work_dir, vol->label->barcode, vol->index->generation); + if (id) { + if (HAVE_BARCODE(vol)) + ret = asprintf(&path, "%s/%s-%d-%c.schema", work_dir, vol->label->barcode, vol->index->generation, id); else - ret = asprintf(&path, "%s/%s-%d.schema", work_dir, vol->label->vol_uuid, vol->index->generation); + ret = asprintf(&path, "%s/%s-%d-%c.schema", work_dir, vol->label->vol_uuid, vol->index->generation, id); } else { - if (strcmp(vol->label->barcode, " ")) + if (HAVE_BARCODE(vol)) ret = asprintf(&path, "%s/%s.schema", work_dir, vol->label->barcode); else ret = asprintf(&path, "%s/%s.schema", work_dir, vol->label->vol_uuid); @@ -2549,13 +2812,8 @@ int ltfs_save_index_to_disk(const char *work_dir, char * reason, bool need_gen, return -ENOMEM; } - if (vol->label->barcode[0] != ' ') { - ltfsmsg(LTFS_INFO, 17235I, vol->label->barcode, 'Z', "Volume Cache", - (unsigned long long)vol->index->file_count, path); - } else { - ltfsmsg(LTFS_INFO, 17235I, LTFS_NO_BARCODE, 'Z', "Volume Cache", - (unsigned long long)vol->index->file_count, path); - } + ltfsmsg(LTFS_INFO, 17235I, _get_barcode(vol), id ? id : 'Z', "Volume Cache", + (unsigned long long)vol->index->file_count, path); ret = xml_schema_to_file(path, vol->index->creator, reason, vol->index); if (ret < 0) { @@ -2566,15 +2824,17 @@ int ltfs_save_index_to_disk(const char *work_dir, char * reason, bool need_gen, } /* Change index file's mode */ - if (chmod(path, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) { + if (chmod(path, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)) { ret = -errno; ltfsmsg(LTFS_ERR, 17184E, errno); } - if (vol->label->barcode[0] != ' ') - ltfsmsg(LTFS_INFO, 17236I, vol->label->barcode, 'Z', path); - else - ltfsmsg(LTFS_INFO, 17236I, LTFS_NO_BARCODE, 'Z', path); + ltfsmsg(LTFS_INFO, 17236I, + _get_barcode(vol), + (unsigned long long)vol->index->generation, + id ? id : 'Z', + (unsigned long long)vol->index->selfptr.block, + path); free(path); return ret; @@ -2670,7 +2930,7 @@ int ltfs_set_barcode(const char *barcode, struct ltfs_volume *vol) } strcpy(vol->label->barcode, barcode); } else - strcpy(vol->label->barcode, " "); + strcpy(vol->label->barcode, NO_BARCODE); return 0; } @@ -2834,9 +3094,12 @@ int ltfs_write_label(tape_partition_t partition, struct ltfs_volume *vol) * the device. * @param vol LTFS volume. The label structure receives a new UUID and format time; all other * label fields should be filled in correctly before calling this function. + * @param density_code density code to format. Used in IBM enterprise tape + * @param destructive use destructive format in MEDIUM_FORMAT command. The FORMAT field is + * specified to 2h. This might trigger the media optimization in LTO9 or later * @return 0 on success or a negative value on error. */ -int ltfs_format_tape(struct ltfs_volume *vol, int density_code) +int ltfs_format_tape(struct ltfs_volume *vol, int density_code, bool destructive) { int ret; struct tc_drive_param cart_param; @@ -2916,8 +3179,11 @@ int ltfs_format_tape(struct ltfs_volume *vol, int density_code) /* Format the tape */ INTERRUPTED_RETURN(); - ltfsmsg(LTFS_INFO, 11097I); - ret = tape_format(vol->device, ltfs_part_id2num(vol->label->partid_ip, vol), density_code); + if (destructive) + ltfsmsg(LTFS_INFO, 17290I); + else + ltfsmsg(LTFS_INFO, 11097I); + ret = tape_format(vol->device, ltfs_part_id2num(vol->label->partid_ip, vol), density_code, destructive); if (ret < 0) { ltfsmsg(LTFS_ERR, 11098E, ret); return ret; @@ -2951,7 +3217,8 @@ int ltfs_format_tape(struct ltfs_volume *vol, int density_code) if (ret < 0) return ret; ltfsmsg(LTFS_INFO, 11278I, vol->label->partid_dp); /* "Writing Index to ..." */ - ret = ltfs_write_index(vol->label->partid_dp, SYNC_FORMAT, vol); + ltfs_set_commit_message_reason(SYNC_FORMAT, vol); + ret = ltfs_write_index(vol->label->partid_dp, SYNC_FORMAT, LTFS_FULL_INDEX, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 11279E, vol->label->partid_dp, ret); return ret; @@ -2964,7 +3231,7 @@ int ltfs_format_tape(struct ltfs_volume *vol, int density_code) if (ret < 0) return ret; ltfsmsg(LTFS_INFO, 11278I, vol->label->partid_ip); /* "Writing Index to ..." */ - ret = ltfs_write_index(vol->label->partid_ip, SYNC_FORMAT, vol); + ret = ltfs_write_index(vol->label->partid_ip, SYNC_FORMAT, LTFS_FULL_INDEX, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 11279E, vol->label->partid_ip, ret); return ret; @@ -2979,9 +3246,11 @@ int ltfs_format_tape(struct ltfs_volume *vol, int density_code) * the device. * @param vol LTFS volume. * @param long_wipe invoke long erase after un-partitioning + * @param destructive use destructive unformat in MEDIUM_FORMAT command. The FORMAT field is + * specified to 2h. This might trigger the media optimization in LTO9 or later * @return 0 on success or a negative value on error. */ -int ltfs_unformat_tape(struct ltfs_volume *vol, bool long_wipe) +int ltfs_unformat_tape(struct ltfs_volume *vol, bool long_wipe, bool destructive) { int ret; @@ -3007,8 +3276,14 @@ int ltfs_unformat_tape(struct ltfs_volume *vol, bool long_wipe) /* Unformat the tape */ INTERRUPTED_RETURN(); - ltfsmsg(LTFS_INFO, 17071I); - ret = tape_unformat(vol->device); + if (destructive) { + ltfsmsg(LTFS_INFO, 17291I); + ret = tape_unformat_hard(vol->device); + } + else { + ltfsmsg(LTFS_INFO, 17071I); + ret = tape_unformat(vol->device); + } if (ret < 0) { ltfsmsg(LTFS_ERR, 17072E, ret); return ret; @@ -3157,10 +3432,7 @@ int ltfs_revalidate(bool have_write_lock, struct ltfs_volume *vol) CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); - if (vol->label->barcode[0] != ' ') - ltfsmsg(LTFS_INFO, 11312I, vol->label->barcode); - else - ltfsmsg(LTFS_INFO, 11312I, LTFS_NO_BARCODE); + ltfsmsg(LTFS_INFO, 11312I, _get_barcode(vol)); /* Block other libltfs operations until revalidation finishes */ ltfs_thread_mutex_lock(&vol->reval_lock); @@ -3349,15 +3621,9 @@ int ltfs_revalidate(bool have_write_lock, struct ltfs_volume *vol) releasewrite_mrsw(&vol->lock); if (ret < 0) { - if (vol->label->barcode[0] != ' ') - ltfsmsg(LTFS_ERR, 11313E, ret, vol->label->barcode); - else - ltfsmsg(LTFS_ERR, 11313E, ret, LTFS_NO_BARCODE); + ltfsmsg(LTFS_ERR, 11313E, ret, _get_barcode(vol)); } else { - if (vol->label->barcode[0] != ' ') - ltfsmsg(LTFS_INFO, 11340I, vol->label->barcode); - else - ltfsmsg(LTFS_INFO, 11340I, LTFS_NO_BARCODE); + ltfsmsg(LTFS_INFO, 11340I, _get_barcode(vol)); } return ret; @@ -3366,11 +3632,13 @@ int ltfs_revalidate(bool have_write_lock, struct ltfs_volume *vol) /** * Write index to tape if the index is dirty, and if there is space available * on the data partition. - * @param vol LTFS volume + * @param reason the reason to write an index * @param index_locking Take index lock while writing an index + * @param type index type to write + * @param vol LTFS volume * @return 0 on success or a negative value on error */ -int ltfs_sync_index(char *reason, bool index_locking, struct ltfs_volume *vol) +int ltfs_sync_index(char *reason, bool index_locking, enum ltfs_index_type type, struct ltfs_volume *vol) { int ret = 0, ret_r = 0; bool dirty; @@ -3378,6 +3646,10 @@ int ltfs_sync_index(char *reason, bool index_locking, struct ltfs_volume *vol) bool dp_index_file_end, ip_index_file_end; char *bc_print = NULL; +#ifndef FORMAT_SPEC25 + type = LTFS_FULL_INDEX; +#endif + start: ret = ltfs_get_partition_readonly(ltfs_dp_id(vol), vol); if (ret < 0 && ret != -LTFS_LESS_SPACE) @@ -3399,10 +3671,7 @@ int ltfs_sync_index(char *reason, bool index_locking, struct ltfs_volume *vol) releaseread_mrsw(&vol->lock); if (dirty) { - if (vol->label->barcode[0] == ' ' || vol->label->barcode == NULL) - bc_print = LTFS_NO_BARCODE; - else - bc_print = vol->label->barcode; + bc_print = _get_barcode(vol); ltfsmsg(LTFS_INFO, 11338I, bc_print, vol->device->serial_number); @@ -3435,12 +3704,27 @@ int ltfs_sync_index(char *reason, bool index_locking, struct ltfs_volume *vol) releasewrite_mrsw(&vol->lock); return ret; } - ret = ltfs_write_index(partition, reason, vol); + ret = ltfs_write_index(partition, reason, type, vol); if (IS_WRITE_PERM(-ret) && partition == ltfs_dp_id(vol)) { - ret_r = ltfs_write_index(ltfs_ip_id(vol), SYNC_WRITE_PERM, vol); + /* + * TODO: Need to determine the last record on DP of the tape and cleanup + * all extents on the volume. Because this write perm have a chance + * to happen in the cases below. + * + * 1. File contents lastly written (and got a write perm on tape drive's flush) + * 2. Index written in ltfs_write_index() + * + * Extents cleaning is required in case1, otherwise LTFS writes an index + * which has unwritten record on the tape. + * + * For now, write the same index on IP and return an error against a sync + * request. + */ + ltfs_set_commit_message_reason(SYNC_WRITE_PERM, vol); + ret_r = ltfs_write_index(ltfs_ip_id(vol), SYNC_WRITE_PERM, LTFS_FULL_INDEX, vol); if (!ret_r) { ltfsmsg(LTFS_INFO, 11344I, bc_print); - ret = ret_r; + ret = -LTFS_SYNC_FAIL_ON_DP; } else { ltfsmsg(LTFS_ERR, 11345E, bc_print); ltfsmsg(LTFS_ERR, 11346E, bc_print); @@ -3474,13 +3758,14 @@ int ltfs_sync_index(char *reason, bool index_locking, struct ltfs_volume *vol) * @param vol LTFS volume * @param partition partition to traverse * @param gen generation to search. 0 to search all + * @param skip_dir skip parsing directory * @param func call back function to call when an index is find. NULL not to call * @param list this pointer is specified as 3rd arguments of call back function * @param priv private data for call back function * @return 0 on success or a negative value on error */ int ltfs_traverse_index_no_eod(struct ltfs_volume *vol, char partition, unsigned int gen, - f_index_found func, void **list, void* priv) + bool skip_dir, f_index_found func, void **list, void* priv) { int ret, func_ret; @@ -3493,7 +3778,7 @@ int ltfs_traverse_index_no_eod(struct ltfs_volume *vol, char partition, unsigned while (true) { ltfs_index_free(&vol->index); ltfs_index_alloc(&vol->index, vol); - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, skip_dir, vol); if (ret < 0 && ret != -LTFS_UNSUPPORTED_INDEX_VERSION) { ltfsmsg(LTFS_ERR, 17075E, 'N', (int)vol->device->position.block, partition); return ret; @@ -3547,13 +3832,14 @@ int ltfs_traverse_index_no_eod(struct ltfs_volume *vol, char partition, unsigned * @param vol LTFS volume * @param partition partition to traverse * @param gen generation to search. 0 to search all + * @param skip_dir skip parsing directory * @param func call back function to call when an index is find. NULL not to call * @param list this pointer is specified as 3rd arguments of call back function * @param priv private data for call back function * @return 0 on success or a negative value on error */ int ltfs_traverse_index_forward(struct ltfs_volume *vol, char partition, unsigned int gen, - f_index_found func, void **list, void* priv) + bool skip_dir, f_index_found func, void **list, void* priv) { int ret, func_ret; struct tape_offset last_index; @@ -3576,7 +3862,7 @@ int ltfs_traverse_index_forward(struct ltfs_volume *vol, char partition, unsigne while (last_index.block >= vol->device->position.block) { ltfs_index_free(&vol->index); ltfs_index_alloc(&vol->index, vol); - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, skip_dir, vol); if (ret < 0 && ret != -LTFS_UNSUPPORTED_INDEX_VERSION) { ltfsmsg(LTFS_ERR, 17075E, 'F', (int)vol->device->position.block, partition); return ret; @@ -3631,12 +3917,13 @@ int ltfs_traverse_index_forward(struct ltfs_volume *vol, char partition, unsigne * @param vol LTFS volume * @param partition partition to traverse * @param gen generation to search. 0 to search all + * @param skip_dir skip parsing directory * @param func call back function to call when an index is find. NULL not to call * @param list this pointer is specified as 3rd arguments of call back function * @return 0 on success or a negative value on error */ int ltfs_traverse_index_backward(struct ltfs_volume *vol, char partition, unsigned int gen, - f_index_found func, void **list, void* priv) + bool skip_dir, f_index_found func, void **list, void* priv) { int ret, func_ret; @@ -3652,7 +3939,7 @@ int ltfs_traverse_index_backward(struct ltfs_volume *vol, char partition, unsign ltfs_index_free(&vol->index); ltfs_index_alloc(&vol->index, vol); - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, skip_dir, vol); if (ret < 0 && ret != -LTFS_UNSUPPORTED_INDEX_VERSION) { ltfsmsg(LTFS_ERR, 17075E, 'B', (int)vol->device->position.block, partition); return ret; @@ -3823,7 +4110,7 @@ static int _ltfs_detect_final_rec_dp(struct ltfs_volume *vol, struct tc_position INTERRUPTED_RETURN(); ltfsmsg(LTFS_INFO, 17120I, "DP", (unsigned long long)seekpos.partition, (unsigned long long)seekpos.block); - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, false, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 17121E, "DP", ret); return ret; @@ -3999,7 +4286,7 @@ int ltfs_recover_eod(struct ltfs_volume *vol) * In index partition, Index will be overwritten. * Read an index only current partition is DP. */ - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, false, vol); if (ret < 0) return ret; } @@ -4040,9 +4327,63 @@ int ltfs_release_medium(struct ltfs_volume *vol) return 0; } +/** + * Capture log page. + * + * At buffer shortage, this function returns the length of the page. The page is truncated + * by size. + * + * @param vol LTFS volume + * @param page log page to capture + * @param subpage subpage to capture + * @param buf buffer of the contents of logpage + * @param size buffer size + * @return page length on success or a negative value on error + */ +int ltfs_logpage(const uint8_t page, const uint8_t subpage, unsigned char *buf, + const size_t size, struct ltfs_volume *vol) +{ + int ret = -EDEV_UNKNOWN; + + if (vol->device) { + tape_device_lock(vol->device); + ret = tape_logsense(vol->device, page, subpage, buf, size); + tape_device_unlock(vol->device); + } + + return ret; +} + +/** + * Capture MAM. + * + * At buffer shortage, this function returns the length of the page. The page is truncated + * by size. + * + * @param vol LTFS volume + * @param part partition to capture + * @param buf buffer of the contents of logpage + * @param size buffer size + * @return MAM length on success or a negative value on error + */ +int ltfs_mam(const tape_partition_t part, unsigned char *buf, + const size_t size, struct ltfs_volume *vol) +{ + int ret = -EDEV_UNKNOWN; + + if (vol->device) { + tape_device_lock(vol->device); + ret = tape_read_attr(vol->device, part, buf, size); + tape_device_unlock(vol->device); + } + + return ret; +} + /** * Wait the drive goes to ready state - * @param device handle to tape device + * + * @param vol LTFS volume * @return 0 on success or a negative value on error */ int ltfs_wait_device_ready(struct ltfs_volume *vol) @@ -4124,7 +4465,7 @@ int ltfs_print_device_list(struct tape_ops *ops) } } - ret = 0; + //ret = 0; return ret; } @@ -4175,3 +4516,267 @@ int ltfs_profiler_set(uint64_t source, struct ltfs_volume *vol) return ret; } + +static int _ltfs_write_rao_file(char *file_path_org, unsigned char *buf, size_t len) +{ + int ret = -EDEV_UNKNOWN; + char *path; + int fd = -1; + ssize_t size = 0; + + ret = asprintf(&path, "%s%s", file_path_org, LTFS_OUT_FILE_EXTENSION); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, __FILE__); + return -LTFS_NO_MEMORY; + } + + fd = open(path, + O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, + S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + if (fd < 0) { + ltfsmsg(LTFS_INFO, 17276I, path, errno); + free(path); + ret = -errno; + return ret; + } + + size = write(fd, buf, len); + if (size < 0) { + ltfsmsg(LTFS_INFO, 17277I, path, errno); + ret = -errno; + goto out; + } else if (size != (ssize_t)len) { + ltfsmsg(LTFS_INFO, 17278I, path, size, (ssize_t)len); + ret = LTFS_FILE_ERR; + goto out; + } else { + ret = 0; + } + + fsync(fd); /* Make sure the data is written to the device */ + +out: + free(path); + close(fd); + return ret; +} + +static int _ltfs_read_rao_file(char *file_path, unsigned char *buf, + size_t len, uint32_t *in_size) +{ + int ret = -EDEV_UNKNOWN; + char *path; + int fd = -1; + struct stat sbuf; + ssize_t size = 0; + + ret = asprintf(&path, "%s", file_path); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, __FILE__); + return -LTFS_NO_MEMORY; + } + + fd = open(path, O_RDONLY | O_BINARY); + if (fd < 0) { + ltfsmsg(LTFS_INFO, 17279I, path, errno); + free(path); + ret = -errno; + return ret; + } + + ret = fstat(fd, &sbuf); + if (ret < 0) { + ret = -errno; + ltfsmsg(LTFS_INFO, 17280I, path, errno); + goto out; + } + + size = read(fd, buf, len); + if (size < 0) { + ltfsmsg(LTFS_INFO, 17281I, path, errno); + ret = -errno; + goto out; + } if (size != (ssize_t)sbuf.st_size) { + ltfsmsg(LTFS_INFO, 17282I, path, size, (ssize_t)sbuf.st_size); + ret = LTFS_FILE_ERR; + goto out; + } else { + ret = 0; + *in_size = (uint32_t)size; + } + +out: + free(path); + close(fd); + return ret; +} + +/** + * Perform the RAO commands and save the results in a .out file. + * @param path The file location of the GRAO buffer to handle. + * @param vol LTFS volume. The label structure receives a new UUID and format time; all other + * label fields should be filled in correctly before calling this function. + */ +int ltfs_get_rao_list(char *path, struct ltfs_volume *vol) +{ + int ret = -EDEV_UNKNOWN; + struct rao_mod rao; + + CHECK_ARG_NULL(path, -LTFS_NULL_ARG); + CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); + + memset(&rao, 0, sizeof(struct rao_mod)); + rao.in_buf = calloc(1, RAO_MAX_RET_SIZE); + if (!rao.in_buf) { + ltfsmsg(LTFS_ERR, 10001E, "ltfs_get_rao_list: out_buf"); + return -ENOMEM; + } + + rao.out_buf = calloc(1, RAO_MAX_RET_SIZE); + if (!rao.out_buf) { + ltfsmsg(LTFS_ERR, 10001E, "ltfs_get_rao_list: out_buf"); + free(rao.in_buf); + return -ENOMEM; + } + + rao.buf_size = RAO_MAX_RET_SIZE; + + ret = tape_device_lock(vol->device); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 12010E, __FUNCTION__); + return ret; + } + + /* get rao */ + ret = _ltfs_read_rao_file(path, rao.in_buf, RAO_MAX_RET_SIZE, &rao.in_size); + if (ret < 0) { + goto out; + } + + /* Send RAO request*/ + ret = tape_rao_request(vol->device, &rao); + if (ret < 0) { + goto out; + } + + /* write data to file */ + ret = _ltfs_write_rao_file(path, rao.out_buf, rao.out_size); + +out: + tape_device_unlock(vol->device); + return ret; +} + +static inline char* _lay_dirs(char *p, struct dentry *d) +{ + int ret = 0; + char *path = NULL; + + if (p) { + if (d->parent) + ret = asprintf(&path, "%s/%s", d->name.name, p); + else { + /* + * root dir doesn't have the parent and d->name.name is "/". + * So directory separator, '/', is not needed. + */ + ret = asprintf(&path, "%s%s", d->name.name, p); + } + free(p); + } else { + ret = asprintf(&path, "%s", d->name.name); + } + + if (ret < 0) + return NULL; + + if (d->parent) { + path = _lay_dirs(path, d->parent); + if (!path) { + /* Memory allocation error */ + return NULL; + } + } + + return path; +} + +/** + * Build full path from a dentry + */ +int ltfs_build_fullpath(char **dest, struct dentry *d) +{ + int ret = 0; + char *path = NULL; + + CHECK_ARG_NULL(dest, -LTFS_NULL_ARG); + + path = _lay_dirs(path, d); + if (path) { + *dest = path; + } else { + ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); + *dest = NULL; + ret = -LTFS_NO_MEMORY; + } + + return ret; +} + +/** + * Update commit message of index to the prefixed message based on sync reason. + * @param reason sync reason defined in ltfs.h + * @param vol LTFS colume + */ +void ltfs_set_commit_message_reason(char *reason, struct ltfs_volume *vol) +{ + ltfs_mutex_lock(&vol->index->dirty_lock); + ltfs_set_commit_message_reason_unlocked(reason, vol); + ltfs_mutex_unlock(&vol->index->dirty_lock); + + return; +} + +/** + * Update commit message of index to the prefixed message based on sync reason. Caller need to + * take index->dirty_lock before calling this function. + * @param reason sync reason defined in ltfs.h + * @param vol LTFS volume + */ +void ltfs_set_commit_message_reason_unlocked(char *reason, struct ltfs_volume *vol) +{ + bool dirty = false; + int ret = 0; + char *str_now = NULL, *msg = NULL; + struct ltfs_timespec now; + + if (!vol->index->dirty) { + /* Do nothing because no update was made */ + goto out; + } + + if (vol->index->commit_message) { + free(vol->index->commit_message); + dirty = true; + } + + ret = get_current_timespec(&now); + if (ret) + goto out; + + ret = xml_format_time(now, &str_now); + if (ret) + goto out; + + ret = asprintf(&msg, "%s - %s", reason, str_now); + if (ret) { + vol->index->commit_message = msg; + dirty = true; + } + +out: + if (dirty) + ltfs_set_index_dirty(false, false, vol->index); + + return; +} diff --git a/src/libltfs/ltfs.h b/src/libltfs/ltfs.h index 1d411dd2..ec518f1a 100644 --- a/src/libltfs/ltfs.h +++ b/src/libltfs/ltfs.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ #ifndef __ltfs_h__ @@ -61,6 +65,10 @@ extern "C" { #include "arch/win/win_util.h" #endif +/* O_BINARY is defined only in MinGW */ +#ifndef O_BINARY +#define O_BINARY 0 +#endif #include #include @@ -146,16 +154,34 @@ struct device_data; #define LTFS_LABEL_VERSION_MIN MAKE_LTFS_VERSION(1,0,0) /* Min supported label version */ #define LTFS_LABEL_VERSION_MAX MAKE_LTFS_VERSION(2,99,99) /* Max supported label version */ + +#ifdef FORMAT_SPEC25 +#define LTFS_LABEL_VERSION MAKE_LTFS_VERSION(2,5,0) /* Written label version */ +#define LTFS_LABEL_VERSION_STR "2.5.0" /* Label version string */ +#else #define LTFS_LABEL_VERSION MAKE_LTFS_VERSION(2,4,0) /* Written label version */ #define LTFS_LABEL_VERSION_STR "2.4.0" /* Label version string */ +#endif #define LTFS_INDEX_VERSION_MIN MAKE_LTFS_VERSION(1,0,0) /* Min supported index version */ #define LTFS_INDEX_VERSION_MAX MAKE_LTFS_VERSION(2,99,99) /* Max supported index version */ + +#ifdef FORMAT_SPEC25 +#define LTFS_INDEX_VERSION MAKE_LTFS_VERSION(2,5,0) /* Written index version */ +#define LTFS_INDEX_VERSION_STR "2.5.0" /* Index version string */ +#else #define LTFS_INDEX_VERSION MAKE_LTFS_VERSION(2,4,0) /* Written index version */ #define LTFS_INDEX_VERSION_STR "2.4.0" /* Index version string */ +#endif #define INDEX_MAX_COMMENT_LEN 65536 /* Maximum comment field length (per LTFS Format) */ +enum ltfs_index_type { + LTFS_INDEX_AUTO = 0, /**< Select index type to write based on specified options */ + LTFS_FULL_INDEX, /**< Forcibly write full index */ + LTFS_INCREMENTAL_INDEX /**< Forcibly write incremental index */ +}; + #define LTFS_NO_BARCODE "NO_BARCODE" #ifndef __APPLE_MAKEFILE__ @@ -168,6 +194,14 @@ struct device_data; #define LTFS_LIVELINK_EA_NAME "ltfs.vendor.IBM.prefixLength" +/* rao parameters */ +#define LTFS_OUT_FILE_EXTENSION ".out" +#define RAO_MAX_FILENUM 2700 /* Maximum file number allowed for RAO */ +#define LTFS_GEOMETORY_OFF (0x00) +#define LTFS_GEOMETORY_ON (0x01) + /* Maximum size allowed that is returned from RAO */ +#define RAO_MAX_RET_SIZE ((RAO_MAX_FILENUM * (32 + (LTFS_GEOMETORY_OFF * 20))) + 8) + #define INTERRUPTED_GOTO(rc, label) \ do{ \ if (ltfs_is_interrupted()) { \ @@ -192,6 +226,9 @@ struct device_data; do {sleep(1);}while(1); \ }while(0) +#define NO_BARCODE " " +#define HAVE_BARCODE(v) strcmp(v->label->barcode, NO_BARCODE) + /* Callback prototype used to list directories. The function must return 0 on success * or a negative value on error. */ typedef int (*ltfs_dir_filler) (void *buf, const char *name, void *priv); @@ -320,7 +357,7 @@ struct dentry { /* Take the iosched_lock before accessing iosched_priv. */ void *iosched_priv; /**< I/O scheduler private data. */ - struct name_list *child_list; /* for hash search */ + struct name_list *child_list; /**< Child list for hash search */ }; struct tape_attr { @@ -344,7 +381,7 @@ typedef enum mam_advisory_lock_status { PWE_MAM_IP = 5, /* Single write perm on IP (Set VOL_IP_PERM__ERR) */ PWE_MAM_BOTH = 6, /* Double write perm (Set both VOL_DP_PERM_ERR and VOL_IP_PERM_ERR) */ NOLOCK_MAM = 128, /* From HPE */ -} mam_lockval; +} mam_lockval_t; #define IS_SINGLE_WRITE_PERM(stat) (stat == PWE_MAM || (stat == PWE_MAM_DP || stat == PWE_MAM_IP) ) #define IS_DOUBLE_WRITE_PERM(stat) (stat == PWE_MAM_BOTH) @@ -367,9 +404,10 @@ enum volumelock_status { }; enum volume_mount_type { - MOUNT_NORMAL = 0, /**< Normal mount */ - MOUNT_ROLLBACK, /**< Roll back mount */ - MOUNT_ERR_TAPE, /**< Mount write perm tape */ + MOUNT_NORMAL = 0, /**< Normal mount */ + MOUNT_ROLLBACK, /**< Roll back mount */ + MOUNT_ROLLBACK_META, /**< Roll back mount only with meta-data */ + MOUNT_ERR_TAPE, /**< Mount write perm tape */ }; #define VOL_WRITE_PERM_MASK (0xE0) @@ -385,7 +423,8 @@ struct ltfs_volume { struct tc_coherency dp_coh; /**< Data partition coherency info */ struct ltfs_label *label; /**< Information from the partition labels */ struct ltfs_index *index; /**< Current cartridge index */ - char *index_cache_path; /**< File name of on-disk index cache */ + char *index_cache_path_w; /**< File name of on-disk index cache update at writing an index */ + char *index_cache_path_r; /**< File name of on-disk index cache update at parsing an index */ /* Opaque handles to higher-level structures */ void *iosched_handle; /**< Handle to the I/O scheduler state */ @@ -435,11 +474,17 @@ struct ltfs_volume { char *mountpoint; /**< Store mount point for Live Link (SDE) */ size_t mountpoint_len; /**< Store mount point path length (SDE) */ struct tape_attr *t_attr; /**< Tape Attribute data */ - mam_lockval lock_status; /**< Total volume lock status from t_attr->vollock and index->vollock */ + mam_lockval_t lock_status; /**< Total volume lock status from t_attr->vollock and index->vollock */ struct ltfs_timespec first_locate; /**< Time to first locate */ - int file_open_count; /**< Number of opened files */ + int file_open_count; /**< Number of opened files */ - const char *work_directory; + /* Journal structure for incremental index */ + struct jentry *journal; /**< Journal for incremental index */ + TAILQ_HEAD(jcreated_struct, jcreated_entry) created_dirs; + bool journal_err; /**< Journal error flag, write a full index next time forcibly */ + + /* Misc */ + const char *work_directory; /**< work directory for profiler data, dump etc.*/ }; struct ltfs_label { @@ -493,10 +538,10 @@ struct ltfs_index { struct index_criteria index_criteria; /**< Active index criteria */ struct dentry *root; /**< The directory tree */ - ltfs_mutex_t rename_lock; /**< Controls name tree access during renames */ + ltfs_mutex_t rename_lock; /**< Controls name tree access during renames */ /* Update tracking */ - ltfs_mutex_t dirty_lock; /**< Controls access to the update tracking bits */ + ltfs_mutex_t dirty_lock; /**< Controls access to the update tracking bits */ bool dirty; /**< Set on metadata update, cleared on write to tape */ bool atime_dirty; /**< Set on atime update, cleared on write to tape */ bool use_atime; /**< Set if atime updates should make the index dirty */ @@ -515,7 +560,13 @@ struct ltfs_index { size_t symerr_count; /**< Number of conflicted symlink dentries */ struct dentry **symlink_conflict; /**< symlink/extent conflicted dentries */ - mam_lockval vollock; /**< volume lock status on index */ + mam_lockval_t vollock; /**< volume lock status on index */ + + /* Incremental index */ + uint64_t full_index_interval; /**< Number of indexes between full indexes */ + uint64_t full_index_to_go; /**< How many incremental index shall be written to the next full index */ + struct tape_offset selfptr_inc; /**< self-pointer of previous incremental index (to prior generation on data partition) */ + struct tape_offset backptr_inc; /**< back pointer of previous incremental index (to prior generation on data partition) */ }; struct ltfs_direntry { @@ -633,6 +684,7 @@ bool ltfs_get_criteria_allow_update(struct ltfs_volume *vol); int ltfs_start_mount(bool trial, struct ltfs_volume *vol); int ltfs_mount(bool force_full, bool deep_recovery, bool recover_extra, bool recover_symlink, unsigned short gen, struct ltfs_volume *vol); +int ltfs_mount_indexfile(char* filename, bool label_check, struct ltfs_volume *vol); int ltfs_unmount(char *reason, struct ltfs_volume *vol); void ltfs_dump_tree_unlocked(struct ltfs_index *index); void ltfs_dump_tree(struct ltfs_volume *vol); @@ -645,8 +697,8 @@ int ltfs_set_volume_name(const char *volname, struct ltfs_volume *vol); int ltfs_set_partition_map(char dp, char ip, int dp_num, int ip_num, struct ltfs_volume *vol); int ltfs_reset_capacity(bool reset, struct ltfs_volume *vol); int ltfs_write_label(tape_partition_t partition, struct ltfs_volume *vol); -int ltfs_format_tape(struct ltfs_volume *vol, int density_code); -int ltfs_unformat_tape(struct ltfs_volume *vol, bool long_erase); +int ltfs_format_tape(struct ltfs_volume *vol, int density_code, bool destructive); +int ltfs_unformat_tape(struct ltfs_volume *vol, bool long_erase, bool destructive); bool ltfs_is_dirty(struct ltfs_volume *vol); int ltfs_load_all_attributes(struct ltfs_volume *vol); @@ -669,25 +721,28 @@ int ltfs_parse_library_backend_opts(void *opt_args, void *opts); void ltfs_set_index_dirty(bool locking, bool atime, struct ltfs_index *idx); void ltfs_unset_index_dirty(bool update_version, struct ltfs_index *idx); -int ltfs_write_index(char partition, char *reason, struct ltfs_volume *vol); -int ltfs_save_index_to_disk(const char *work_dir, char * reason, bool need_gen, struct ltfs_volume *vol); - +int ltfs_write_index(char partition, char *reason, enum ltfs_index_type type, struct ltfs_volume *vol); +int ltfs_save_index_to_disk(const char *work_dir, char *reason, char id, struct ltfs_volume *vol); char ltfs_dp_id(struct ltfs_volume *vol); char ltfs_ip_id(struct ltfs_volume *vol); const char *ltfs_get_volume_uuid(struct ltfs_volume *vol); -int ltfs_sync_index(char *reason, bool index_locking, struct ltfs_volume *vol); +int ltfs_sync_index(char *reason, bool index_locking, enum ltfs_index_type type, struct ltfs_volume *vol); int ltfs_traverse_index_forward(struct ltfs_volume *vol, char partition, unsigned int gen, - f_index_found func, void **list, void *priv); + bool skip_dir, f_index_found func, void **list, void *priv); int ltfs_traverse_index_backward(struct ltfs_volume *vol, char partition, unsigned int gen, - f_index_found func, void **list, void *priv); + bool skip_dir, f_index_found func, void **list, void *priv); int ltfs_traverse_index_no_eod(struct ltfs_volume *vol, char partition, unsigned int gen, - f_index_found func, void **list, void *priv); + bool skip_dir, f_index_found func, void **list, void *priv); int ltfs_check_eod_status(struct ltfs_volume *vol); int ltfs_recover_eod(struct ltfs_volume *vol); int ltfs_release_medium(struct ltfs_volume *vol); +int ltfs_logpage(const uint8_t page, const uint8_t subpage, unsigned char *buf, + const size_t size, struct ltfs_volume *vol); +int ltfs_mam(const tape_partition_t part, unsigned char *buf, + const size_t size, struct ltfs_volume *vol); int ltfs_wait_device_ready(struct ltfs_volume *vol); void ltfs_recover_eod_simple(struct ltfs_volume *vol); @@ -695,6 +750,13 @@ int ltfs_print_device_list(struct tape_ops *ops); void ltfs_enable_livelink_mode(struct ltfs_volume *vol); int ltfs_profiler_set(uint64_t source, struct ltfs_volume *vol); + +int ltfs_get_rao_list(char *path, struct ltfs_volume *vol); +int ltfs_build_fullpath(char **dest, struct dentry *d); + +void ltfs_set_commit_message_reason(char *reason, struct ltfs_volume *vol); +void ltfs_set_commit_message_reason_unlocked(char *reason, struct ltfs_volume *vol); + #ifdef __cplusplus } #endif diff --git a/src/libltfs/ltfs_error.h b/src/libltfs/ltfs_error.h index a7abe8ce..703b6324 100644 --- a/src/libltfs/ltfs_error.h +++ b/src/libltfs/ltfs_error.h @@ -257,6 +257,67 @@ #define LTFS_TAPE_REMOVED 1198 /* The tape is already removed */ #define LTFS_NEED_MOVE 1199 /* Need to move tape to a storage slot first before the operation */ #define LTFS_NEED_START_OVER 1200 /* Need operation needs to be started again */ +#define LTFS_LOCATE_ERROR 1201 /* Locate returns write-perm error */ +#define LTFS_STATS_DB_OPEN 1202 /* Failed to open a stats DB */ +#define LTFS_NO_TRAIL_FM 1203 /* There is no trailing FM after an index */ +#define LTFS_SAFENAME_FAIL 1204 /* Failed to update safename */ +#define LTFS_SYNC_FAIL_ON_DP 1205 /* Unwritten file contents exists in sync */ + +/* + * Error codes for the XML parser + */ +#define LTFS_XML_READ_FAIL 5000 /* Error happens in reading XML node */ +#define LTFS_XML_CONST_FAIL 5001 /* Error happens in reading XML const value */ +#define LTFS_XML_WRONG_NODE 5002 /* Unexpected node type is detected */ +#define LTFS_XML_UNEXPECTED_EOF 5003 /* Unexpected XML document end is detected */ +#define LTFS_XML_EMPTY_UNKNOWN 5004 /* Cannot detect a tags is empty or not */ +#define LTFS_XML_EMPTY 5005 /* The tag is empty */ +#define LTFS_XML_SKIP_FAIL 5006 /* Error skipping unrecognized tag */ +#define LTFS_XML_NO_REQUIRED_TAG 5007 /* Do not find required tag */ +#define LTFS_XML_DUPLICATED_TAG 5008 /* Duplicated tag is detected */ +#define LTFS_XML_OPEN_TAG 5009 /* XML tag is not closed correctly */ +#define LTFS_XML_SAVE_FAIL 5010 /* Cannot save the tag */ +#define LTFS_XML_WRONG_TOPTAG 5011 /* Unexpected top tag in the XML document */ +#define LTFS_XML_WRONG_ENCODING 5012 /* Unexpected encoding is detected */ +#define LTFS_XML_TOP_ATTR_FAIL 5013 /* Failed to get attribute of the top tag */ +#define LTFS_XML_WRONG_UUID 5014 /* Wrong UUID is detected */ +#define LTFS_XML_WRONG_GEN 5015 /* Cannot parse generation number correctly */ +#define LTFS_XML_WRONG_UTIME 5016 /* Cannot parse index's updatetime correctly */ +#define LTFS_XML_WRONG_LOC 5017 /* Wrong tape position of index or labal is detected */ +#define LTFS_XML_WRONG_LOC_PREV 5018 /* Wrong tape position of previous index is detected */ +#define LTFS_XML_WRONG_PA 5019 /* Unexpected policyupdate value is detected */ +#define LTFS_XML_WRONG_POLICY 5020 /* Wrong policy value is detected */ +#define LTFS_XML_TOO_LONG_COMMENT 5021 /* Too long index comment */ +#define LTFS_XML_WRONG_NEXT 5022 /* Wrong maxuid tag is detected */ +#define LTFS_XML_WRONG_RO_DIR 5023 /* Unexpected readonly value is detected in a dir */ +#define LTFS_XML_WRONG_MTIME_DIR 5024 /* Unexpected timestamp in dir mtime */ +#define LTFS_XML_WRONG_CRTIME_DIR 5025 /* Unexpected timestamp in dir creation time */ +#define LTFS_XML_WRONG_ATIME_DIR 5026 /* Unexpected timestamp in dir atime */ +#define LTFS_XML_WRONG_CTIME_DIR 5027 /* Unexpected timestamp in dir ctime */ +#define LTFS_XML_WRONG_BTIME_DIR 5028 /* Unexpected timestamp in dir backup time */ +#define LTFS_XML_XATTR_TYPE 5029 /* Unexpected xattr type */ +#define LTFS_XML_XATTR_SIZE 5030 /* Unexpected xattr size */ +#define LTFS_XML_WRONG_UID 5031 /* Unexpected UID number */ +#define LTFS_XML_INVALID_UID 5032 /* UID number validation is failed */ +#define LTFS_XML_WRONG_RO_F 5033 /* Unexpected readonly value is detected in a file */ +#define LTFS_XML_WRONG_MTIME_F 5034 /* Unexpected timestamp in file mtime */ +#define LTFS_XML_WRONG_CRTIME_F 5035 /* Unexpected timestamp in file creation time */ +#define LTFS_XML_WRONG_ATIME_F 5036 /* Unexpected timestamp in file atime */ +#define LTFS_XML_WRONG_CTIME_F 5037 /* Unexpected timestamp in file ctime */ +#define LTFS_XML_WRONG_BTIME_F 5038 /* Unexpected timestamp in file backup time */ +#define LTFS_XML_WRONG_SIZE 5039 /* Unexpected file size value */ +#define LTFS_XML_WRONG_PART 5040 /* Unexpected partition ID */ +#define LTFS_XML_WRONG_START_BLK 5041 /* Unexpected startblock */ +#define LTFS_XML_WRONG_OFFSET 5042 /* Unexpected offset */ +#define LTFS_XML_WRONG_BYTE_CNT 5043 /* Unexpected bytecount */ +#define LTFS_XML_WRONG_FILE_OFST 5044 /* Unexpected fileoffset */ +#define LTFS_XML_EXT_OVERLAP 5045 /* Extent list is overlapped */ +#define LTFS_XML_EXT_TOO_LONG 5046 /* Extent list is longer than file size */ +#define LTFS_XML_WRONG_FTIME_L 5047 /* Unexpected timestamp in tape format time in a label */ +#define LTFS_XML_WRONG_PART_MAP 5048 /* Unexpected partition map in a label */ +#define LTFS_XML_WRONG_BLOCKSIZE 5049 /* Unexpected blocksize in a label */ +#define LTFS_XML_WRONG_COMP 5050 /* Unexpected compression in a label */ +#define LTFS_BAD_INDEX_TYPE 5051 /* Unsupported index type is specified */ #define LTFS_ERR_MAX 19999 @@ -458,4 +519,16 @@ #define MKLTFS_USAGE_SYNTAX_ERROR PROG_USAGE_SYNTAX_ERROR /* Wrong argument */ #define MKLTFS_CANCELED_BY_USER PROG_CANCELED_BY_USER /* Canceled by user */ +/* Status code for ltfsindextool + * The maximum return code from the program is 0xFF. + */ +#define INDEXTOOL_NO_ERRORS PROG_NO_ERRORS /* No error and the cartridge is not modified */ +#define INDEXTOOL_CORRECTED PROG_TREAT_SUCCESS /* Recover correctly, the cartridge is modified */ +#define INDEXTOOL_REBOOT_REQUIRED PROG_REBOOT_REQUIRED /* Reboot required */ +#define INDEXTOOL_UNCORRECTED PROG_UNCORRECTED /* Cannot recover, the cartridge is modified */ +#define INDEXTOOL_OPERATIONAL_ERROR PROG_OPERATIONAL_ERROR /* Get device error while processing, the cartridge may be modified */ +#define INDEXTOOL_USAGE_SYNTAX_ERROR PROG_USAGE_SYNTAX_ERROR /* Wrong argument */ +#define INDEXTOOL_CANCELED_BY_USER PROG_CANCELED_BY_USER /* Canceled by user */ +#define INDEXTOOL_SHARED_LIB_ERROR PROG_SHARED_LIB_ERROR /* Library error */ + #endif /* __ltfs_error_h__ */ diff --git a/src/libltfs/ltfs_fsops.c b/src/libltfs/ltfs_fsops.c index 671b2cfe..98868d41 100644 --- a/src/libltfs/ltfs_fsops.c +++ b/src/libltfs/ltfs_fsops.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -62,6 +66,7 @@ #include "dcache.h" #include "pathname.h" #include "index_criteria.h" +#include "inc_journal.h" #include "arch/time_internal.h" int ltfs_fsops_open(const char *path, bool open_write, bool use_iosched, struct dentry **d, @@ -363,9 +368,10 @@ int ltfs_fsops_create(const char *path, bool isdir, bool readonly, bool overwrit if (! isdir) ++vol->index->file_count; ltfs_set_index_dirty(false, false, vol->index); + incj_create(path_norm, d, vol); d->dirty = true; ltfs_mutex_unlock(&vol->index->dirty_lock); - vol->file_open_count ++; + vol->file_open_count++; *dentry = d; ret = 0; @@ -518,8 +524,12 @@ int ltfs_fsops_unlink(const char *path, ltfs_file_id *id, struct ltfs_volume *vo releasewrite_mrsw(&d->meta_lock); ltfs_mutex_lock(&vol->index->dirty_lock); - if (! d->isdir) + if (d->isdir) { + incj_rmdir(path_norm, d, vol); + } else { + incj_rmfile(path_norm, d, vol); --vol->index->file_count; + } ltfs_set_index_dirty(false, false, vol->index); ltfs_mutex_unlock(&vol->index->dirty_lock); @@ -544,7 +554,7 @@ int ltfs_fsops_rename(const char *from, const char *to, ltfs_file_id *id, struct int ret; char *from_norm = NULL, *to_norm = NULL; char *from_norm_copy = NULL, *to_norm_copy = NULL; - char *from_filename, *to_filename; + char *from_filename = NULL, *to_filename = NULL, *to_filename_incj = NULL; char *to_filename_copy = NULL, *to_filename_copy2 = NULL; struct dentry *fromdir = NULL, *todir = NULL; struct dentry *fromdentry = NULL, *todentry = NULL; @@ -583,14 +593,12 @@ int ltfs_fsops_rename(const char *from, const char *to, ltfs_file_id *id, struct goto out_free; } - if (dcache_initialized(vol)) { - from_norm_copy = strdup(from_norm); - to_norm_copy = strdup(to_norm); - if (! from_norm_copy || ! to_norm_copy) { - ltfsmsg(LTFS_ERR, 10001E, "ltfs_fsops_rename: file name copy"); - ret = -LTFS_NO_MEMORY; - goto out_free; - } + from_norm_copy = strdup(from_norm); + to_norm_copy = strdup(to_norm); + if (! from_norm_copy || ! to_norm_copy) { + ltfsmsg(LTFS_ERR, 10001E, "ltfs_fsops_rename: file name copy"); + ret = -LTFS_NO_MEMORY; + goto out_free; } /* Split paths into directory and file name */ @@ -864,6 +872,16 @@ int ltfs_fsops_rename(const char *from, const char *to, ltfs_file_id *id, struct fromdentry->dirty = true; + /* Process the incremental journal */ + if (fromdentry->isdir) + incj_rmdir(from_norm_copy, fromdentry, vol); + else + incj_rmfile(from_norm_copy, fromdentry, vol); + + fs_split_path(to_norm_copy, &to_filename_incj, strlen(to_norm_copy) + 1); + incj_create(to_norm_copy, fromdentry, vol); + + /* Release dentry of source */ if (! iosched_initialized(vol)) fs_release_dentry_unlocked(fromdentry); else @@ -1037,7 +1055,7 @@ int ltfs_fsops_setxattr(const char *path, const char *name, const char *value, s goto out_free; } - new_name_strip = _xattr_strip_name(new_name); + new_name_strip = xattr_strip_name(new_name); if (! new_name_strip) { /* Namespace is not supported (Linux) */ ret = -LTFS_XATTR_NAMESPACE; @@ -1082,11 +1100,11 @@ int ltfs_fsops_setxattr(const char *path, const char *name, const char *value, s id->uid = d->uid; id->ino = d->ino; - /* Save original value */ - ret_restore = xattr_get(d, new_name_strip, value_restore, sizeof(value_restore), vol); - - ret = xattr_set(d, new_name_strip, value, size, flags, vol); if (dcache_initialized(vol)) { + /* Save original value */ + ret_restore = xattr_get(d, new_name_strip, value_restore, sizeof(value_restore), vol); + + ret = xattr_set(d, new_name_strip, value, size, flags, vol); if (ret == 0) { ret = dcache_setxattr(new_path, d, new_name_strip, value, size, flags, vol); if (ret < 0) { @@ -1097,9 +1115,10 @@ int ltfs_fsops_setxattr(const char *path, const char *name, const char *value, s } } dcache_close(d, true, true, vol); - } else + } else { + ret = xattr_set(d, new_name_strip, value, size, flags, vol); fs_release_dentry(d); - + } if (NEED_REVAL(ret)) { ret = ltfs_revalidate(write_lock, vol); if (ret == 0) @@ -1152,7 +1171,7 @@ int ltfs_fsops_getxattr(const char *path, const char *name, char *value, size_t ltfsmsg(LTFS_ERR, 11125E, ret); goto out_free; } - new_name_strip = _xattr_strip_name(new_name); + new_name_strip = xattr_strip_name(new_name); if (! new_name_strip) { /* Namespace is not supported (Linux) */ ret = -LTFS_NO_XATTR; @@ -1304,7 +1323,7 @@ int ltfs_fsops_removexattr(const char *path, const char *name, ltfs_file_id *id, ltfsmsg(LTFS_ERR, 11137E, ret); goto out_free; } - new_name_strip = _xattr_strip_name(new_name); + new_name_strip = xattr_strip_name(new_name); if (! new_name_strip) { /* Namespace is not supported (Linux) */ ret = -LTFS_NO_XATTR; @@ -1560,7 +1579,7 @@ int ltfs_fsops_utimens(struct dentry *d, const struct ltfs_timespec ts[2], struc d->platform_safe_name, (unsigned long long)d->uid, (unsigned long long)ts[0].tv_sec); get_current_timespec(&d->change_time); ltfs_set_index_dirty(true, true, vol->index); - d->dirty = true; + ltfs_set_dentry_dirty(d, vol); } if (d->modify_time.tv_sec != ts[1].tv_sec || d->modify_time.tv_nsec != ts[1].tv_nsec) { d->modify_time = ts[1]; @@ -1570,7 +1589,7 @@ int ltfs_fsops_utimens(struct dentry *d, const struct ltfs_timespec ts[2], struc d->platform_safe_name, (unsigned long long)d->uid, (unsigned long long)ts[1].tv_sec); get_current_timespec(&d->change_time); ltfs_set_index_dirty(true, false, vol->index); - d->dirty = true; + ltfs_set_dentry_dirty(d, vol); } if (dcache_initialized(vol)) dcache_flush(d, FLUSH_METADATA, vol); @@ -1637,7 +1656,7 @@ int ltfs_fsops_utimens_all(struct dentry *d, const struct ltfs_timespec ts[4], s d->platform_safe_name, (unsigned long long)d->uid, (unsigned long long)ts[3].tv_sec); isctime=true; ltfs_set_index_dirty(true, false, vol->index); - d->dirty = true; + ltfs_set_dentry_dirty(d, vol); } if (ts[0].tv_sec != 0 || ts[0].tv_nsec != 0) { d->access_time = ts[0]; @@ -1647,7 +1666,7 @@ int ltfs_fsops_utimens_all(struct dentry *d, const struct ltfs_timespec ts[4], s d->platform_safe_name, (unsigned long long)d->uid, (unsigned long long)ts[0].tv_sec); if(!isctime) get_current_timespec(&d->change_time); ltfs_set_index_dirty(true, true, vol->index); - d->dirty = true; + ltfs_set_dentry_dirty(d, vol); } if (ts[1].tv_sec != 0 || ts[1].tv_nsec != 0) { d->modify_time = ts[1]; @@ -1657,7 +1676,7 @@ int ltfs_fsops_utimens_all(struct dentry *d, const struct ltfs_timespec ts[4], s d->platform_safe_name, (unsigned long long)d->uid, (unsigned long long)ts[1].tv_sec); if(!isctime) get_current_timespec(&d->change_time); ltfs_set_index_dirty(true, false, vol->index); - d->dirty = true; + ltfs_set_dentry_dirty(d, vol); } if (ts[2].tv_sec != 0 || ts[2].tv_nsec != 0) { d->creation_time = ts[2]; @@ -1667,7 +1686,7 @@ int ltfs_fsops_utimens_all(struct dentry *d, const struct ltfs_timespec ts[4], s d->platform_safe_name, (unsigned long long)d->uid, (unsigned long long)ts[2].tv_sec); if(!isctime) get_current_timespec(&d->change_time); ltfs_set_index_dirty(true, false, vol->index); - d->dirty = true; + ltfs_set_dentry_dirty(d, vol); } if (dcache_initialized(vol)) @@ -1679,7 +1698,6 @@ int ltfs_fsops_utimens_all(struct dentry *d, const struct ltfs_timespec ts[4], s return 0; } - int ltfs_fsops_set_readonly(struct dentry *d, bool readonly, struct ltfs_volume *vol) { int ret; @@ -1705,6 +1723,7 @@ int ltfs_fsops_set_readonly(struct dentry *d, bool readonly, struct ltfs_volume d->readonly = readonly; get_current_timespec(&d->change_time); ltfs_set_index_dirty(true, false, vol->index); + ltfs_set_dentry_dirty(d, vol); if (dcache_initialized(vol)) dcache_flush(d, FLUSH_METADATA, vol); } @@ -1793,6 +1812,10 @@ ssize_t ltfs_fsops_read(struct dentry *d, char *buf, size_t count, off_t offset, if (d->isdir) return -LTFS_ISDIRECTORY; + if (vol->mount_type == MOUNT_ROLLBACK_META) { + return -LTFS_DEVICE_UNREADY; + } + if (iosched_initialized(vol)) ret = iosched_read(d, buf, count, offset, vol); else @@ -1991,86 +2014,87 @@ int ltfs_fsops_readlink_path(const char* path, char* buf, size_t size, ltfs_file int ltfs_fsops_target_absolute_path(const char* link, const char* target, char* buf, size_t size ) { - char *work_buf, *target_buf, *temp_buf, *token, *next_token; // work buffer for string - int len=0,len2=0; // work variables for string length + char *work_buf, *target_buf, *temp_buf, *token, *next_token; /* work buffers for string */ + int len=0, len2=0; /* work variables for string length */ CHECK_ARG_NULL(link, -LTFS_NULL_ARG); CHECK_ARG_NULL(target, -LTFS_NULL_ARG); - // need to set message and return code + /* need to set message and return code */ if (link[0]!='/') { return -LTFS_BAD_ARG; } + /* Check input target string is already absolute path or not */ len2 = strlen(target); - // input target string is already absolute path if ( (target[0]=='/') && !strstr(target,"./" ) ) { if ( size < (size_t)len2+1) { return -LTFS_SMALL_BUFFER; } - strcpy( buf, target ); + strcpy(buf, target); return 0; } len=strlen(link); - work_buf = malloc(len+len2+1); + work_buf = malloc(len + len2 + 1); if (!work_buf) { return -LTFS_NO_MEMORY; } - target_buf = malloc(len2+1); + target_buf = malloc(len2 + 1); if (!target_buf) { - free( work_buf ); + free(work_buf); return -LTFS_NO_MEMORY; } - if ( target[0]=='/' ) { - temp_buf=strstr( target, "/." ); // get "/../ccc/target.txt" of "/aaa/../ccc/target.txt" - strcpy(target_buf,temp_buf+1); // copy "../ccc/target.txt" - len=strlen(target_buf)+1; - len=len2-len; - strncpy( work_buf, target, len ); // copy "/aaa" + if (target[0]=='/') { + temp_buf = strstr(target, "/."); /* get "/../ccc/target.txt" of "/aaa/../ccc/target.txt" */ + strcpy(target_buf, temp_buf + 1); /* copy "../ccc/target.txt" */ + len = strlen(target_buf) + 1; + len = len2 - len; + strncpy(work_buf, target, len); /* copy "/aaa" */ } else { - strcpy( work_buf, link ); - strcpy( target_buf, target ); - - // split link file name then get current directory - temp_buf=strrchr( work_buf, '/' ); // get "/link.txt" from "/aaa/bbb/link.txt" - len = len-strlen(temp_buf); // length of "/aaa/bbb" - } - - // split target path directory then modify current directory with target path information - token = strtok( target_buf,"/" ); // get ".." from "../ccc/target.txt" - while ( token ) { - next_token = strtok( NULL,"/" ); // if next_token is NULL then token is filename - if ( !next_token ) break; - if ( strcmp( token,".." )==0 ) { - work_buf[len]='\0'; // "/aaa/bbb\0link.txt" - temp_buf=strrchr( work_buf, '/' ); // get "/bbb" + strcpy(work_buf, link); + strcpy(target_buf, target); + + /* Split link file name then get current directory */ + temp_buf = strrchr(work_buf, '/'); /* get "/link.txt" from "/aaa/bbb/link.txt" */ + len -= strlen(temp_buf); /* length of "/aaa/bbb" */ + } + + /* Split target path directory then modify current directory with target path information */ + token = strtok(target_buf, "/"); /* get ".." from "../ccc/target.txt" */ + while (token) { + next_token = strtok(NULL, "/"); /* if next_token is NULL then token is filename */ + if (!next_token) + break; + if (strcmp(token, "..") == 0) { + work_buf[len] = '\0'; /* "/aaa/bbb\0link.txt" */ + temp_buf = strrchr(work_buf, '/' ); /* get "/bbb" */ if (!temp_buf) { - *buf = '\0'; // out of ltfs range + *buf = '\0'; /* out of ltfs range */ return 0; } - len = len-strlen(temp_buf); // length of "/aaa" - } else if ( strcmp( token, "." ) ) { // have directory name - work_buf[len]='/'; // put '/ 'as "/aaa/" - strncpy( work_buf+len+1, token, strlen(token)+1 ); // "/aaa/ccc\0" - len=strlen(work_buf); + len -= strlen(temp_buf); /* length of "/aaa" */ + } else if (strcmp(token, "." )) { /* have directory name */ + work_buf[len] = '/'; /* put '/ 'as "/aaa/" */ + strncpy(work_buf+len+1, token, strlen(token)+1); /* "/aaa/ccc\0" */ + len = strlen(work_buf); } token = next_token; } - work_buf[len]='/'; // put '/ 'as "/aaa/ccc/" - strncpy( work_buf+len+1, token, strlen(token)+1 ); // "/aaa/ccc/target.txt\0" + work_buf[len] = '/'; /* put '/ 'as "/aaa/ccc/" */ + strncpy(work_buf+len+1, token, strlen(token)+1); /* "/aaa/ccc/target.txt\0" */ - if ( size < strlen(work_buf)+1 ) { - free( work_buf ); - free( target_buf ); + if (size < strlen(work_buf) + 1) { + free(work_buf); + free(target_buf); return -LTFS_SMALL_BUFFER; } - strcpy( buf, work_buf ); - free( work_buf ); - free( target_buf ); + strcpy(buf, work_buf); + free(work_buf); + free(target_buf); return 0; } @@ -2082,7 +2106,11 @@ int ltfs_fsops_volume_sync(char *reason, struct ltfs_volume *vol) if (ret < 0) return ret; - ret = ltfs_sync_index(reason, true, vol); + ltfs_mutex_lock(&vol->index->dirty_lock); + ltfs_set_commit_message_reason_unlocked(reason, vol); + ltfs_mutex_unlock(&vol->index->dirty_lock); + + ret = ltfs_sync_index(reason, true, LTFS_INDEX_AUTO, vol); return ret; } diff --git a/src/libltfs/ltfs_fsops_raw.c b/src/libltfs/ltfs_fsops_raw.c index 2f6df457..d45a86de 100644 --- a/src/libltfs/ltfs_fsops_raw.c +++ b/src/libltfs/ltfs_fsops_raw.c @@ -398,7 +398,7 @@ int _ltfs_fsraw_add_extent_unlocked(struct dentry *d, struct extent_info *ext, b * No need to mark at this time but reserve this value for fueture release */ d->extents_dirty = true; - d->dirty = true; + ltfs_set_dentry_dirty(d, vol); releasewrite_mrsw(&d->meta_lock); ltfs_set_index_dirty(true, false, vol->index); @@ -443,8 +443,9 @@ int ltfs_fsraw_add_extent(struct dentry *d, struct extent_info *ext, bool update int ltfs_fsraw_cleanup_extent(struct dentry *d, struct tc_position err_pos, unsigned long blocksize, struct ltfs_volume *vol) { int ret = 0; - struct name_list *entry, *tmp; + struct name_list *entry, *tmp; struct extent_info *ext, *preventry; + struct tc_position extent_last = {0, 0, UINT32_MAX, false, false}; if (HASH_COUNT(d->child_list) != 0) { HASH_ITER(hh, d->child_list, entry, tmp) { @@ -453,7 +454,24 @@ int ltfs_fsraw_cleanup_extent(struct dentry *d, struct tc_position err_pos, unsi } else { TAILQ_FOREACH_REVERSE_SAFE(ext, &entry->d->extentlist, extent_struct, list, preventry) { - if (err_pos.block <= (ext->start.block + ext->bytecount/blocksize)) { + if (ext->start.block && ext->bytecount) { + extent_last.partition = ltfs_part_id2num(ext->start.partition, vol); + /* Calculate the last block of this extent */ + extent_last.block = ext->start.block + (ext->bytecount / blocksize); + if ( (ext->bytecount % blocksize) == 0 ) + extent_last.block--; + } else { + extent_last.partition = UINT32_MAX; + extent_last.block = 0; + } + + /* + * err_pos has the first block number that tape drive has it on buffer + * but not transferred to the medium. + * It means position (err_pos-1) is the last block on the medium. + */ + if ( extent_last.partition == err_pos.partition && err_pos.block <= extent_last.block ) { + ltfsmsg(LTFS_INFO, 11334I, entry->name, (unsigned long long)ext->start.block, (unsigned long long)ext->bytecount); ret = ltfs_get_volume_lock(false, vol); @@ -775,12 +793,12 @@ int ltfs_fsraw_truncate(struct dentry *d, off_t length, struct ltfs_volume *vol) d->realsize = new_realsize; get_current_timespec(&d->modify_time); d->change_time = d->modify_time; + ltfs_set_dentry_dirty(d, vol); releasewrite_mrsw(&d->meta_lock); releasewrite_mrsw(&d->contents_lock); ltfs_set_index_dirty(true, false, vol->index); - d->dirty = true; releaseread_mrsw(&vol->lock); return 0; diff --git a/src/libltfs/ltfs_internal.c b/src/libltfs/ltfs_internal.c index 7257e031..028e32bf 100644 --- a/src/libltfs/ltfs_internal.c +++ b/src/libltfs/ltfs_internal.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -44,6 +44,10 @@ ** IBM Almaden Research Center ** bbiskebo@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -61,6 +65,7 @@ #include "iosched.h" #include "ltfs_fsops.h" #include "xattr.h" +#include "inc_journal.h" /** * Allocate an empty LTFS index. @@ -384,11 +389,13 @@ int ltfs_read_one_label(tape_partition_t partition, struct ltfs_label *label, * This function does not read over another file mark * @param eod_pos EOD position for current partition, or 0 to assume that EOD will not be * encountered during parsing. + * @param recover_symlink recover symlink conflict + * @param skip_dir skip parsing directory * @param vol the volume * @return 0 on success, 1 if index file does not end with a file mark (but is otherwise valid), * or a negative value on error. */ -int ltfs_read_index(uint64_t eod_pos, bool recover_symlink, struct ltfs_volume *vol) +int ltfs_read_index(uint64_t eod_pos, bool recover_symlink, bool skip_dir, struct ltfs_volume *vol) { int ret, ret_sym; struct tc_position pos; @@ -410,7 +417,7 @@ int ltfs_read_index(uint64_t eod_pos, bool recover_symlink, struct ltfs_volume * } /* Parse and validate the schema */ - ret = xml_schema_from_tape(eod_pos, vol); + ret = xml_schema_from_tape(eod_pos, skip_dir, vol); if ( vol->index->symerr_count ) { if ( recover_symlink ) { ret_sym = ltfs_split_symlink( vol ); @@ -429,7 +436,7 @@ int ltfs_read_index(uint64_t eod_pos, bool recover_symlink, struct ltfs_volume * if (ret < 0) { ltfsmsg(LTFS_WARN, 11194W, ret); return ret; - } else if (ret == 1) + } else if (ret == LTFS_NO_TRAIL_FM) end_fm = false; /* check volume UUID */ @@ -473,6 +480,49 @@ int ltfs_read_index(uint64_t eod_pos, bool recover_symlink, struct ltfs_volume * return end_fm ? 0 : 1; } +/** + * Read an index file from file, storing the result in the given volume. + * + * @param filename file name of index + * @param recover_symlink recover symlink conflict + * @param vol the volume + * @return 0 on success, or a negative value on error. + */ +int ltfs_read_indexfile(char* filename, bool recover_symlink, struct ltfs_volume *vol) +{ + int ret, ret_sym; + + ltfs_index_free(&vol->index); + ret = ltfs_index_alloc(&vol->index, vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 11297E, ret); + return ret; + } + + /* Parse and validate the schema */ + ret = xml_schema_from_file(filename, vol->index, vol); + if ( vol->index->symerr_count ) { + if ( recover_symlink ) { + ret_sym = ltfs_split_symlink( vol ); + if (ret_sym < 0) { + ret = ret_sym; + } else if (ret == -LTFS_SYMLINK_CONFLICT) { + ret = 0; + } + } else { + ltfsmsg(LTFS_ERR, 11321E); + } + free( vol->index->symlink_conflict ); + vol->index->symerr_count = 0; + } + + if (ret < 0) { + ltfsmsg(LTFS_WARN, 11194W, ret); + } + + return ret; +} + /** * Returns true iff a given char corresponds to a valid logical partition ID. */ @@ -540,7 +590,7 @@ int ltfs_seek_index(char partition, tape_block_t *eod_pos, tape_block_t *index_e /* try to read an index file */ check_err(tape_spacefm(vol->device, 1), 11202E, out); - ret = ltfs_read_index(*eod_pos, recover_symlink, vol); + ret = ltfs_read_index(*eod_pos, recover_symlink, false, vol); if (ret < 0) { /* no index file found: go back 2 file marks and try again */ ltfsmsg(LTFS_DEBUG, 11204D); @@ -704,7 +754,7 @@ int _ltfs_check_pointers(struct ltfs_index *ip_index, struct ltfs_index *dp_inde ret = tape_seek(vol->device, &seekpos); if (ret < 0) return ret; - ret = ltfs_read_index(0, false, vol); + ret = ltfs_read_index(0, false, true, vol); if (ret < 0) return ret; dp_backptr = vol->index->backptr.block; @@ -982,6 +1032,68 @@ int _ltfs_make_lost_found(tape_block_t ip_eod, tape_block_t dp_eod, return 0; } +/** + * Find the appropriate append address for a partition at which calling ltfs_write_index() will + * correctly restore consistency to the partition. + * + * @param vol The ltfs volume to work on + * @param partid The partition to examine + * @param block The starting block of the final index on the partition + * @param fix allow simple fixes to make the volume consistent? + * @return 0 if eod is the appropriate append address, >0 for the absolute block address to append + * at, <0 on error + */ +static int _ltfs_find_append_blk_after_idx(struct ltfs_volume *vol, char partid, tape_block_t block, bool fix) { + unsigned int n_fm_after = 0; + struct tc_position idx_pos; + struct tc_position final_fm_pos; + int ret = 0; + + idx_pos.partition = ltfs_part_id2num(partid, vol); + idx_pos.block = block; + ret = tape_seek(vol->device, &idx_pos); + if (ret != 0) { + if (partid == vol->label->partid_ip) + ltfsmsg(LTFS_ERR, 11023E); + else { + ltfsmsg(LTFS_ERR, 11020E); + } + goto out; + } + while(ret == 0) { + ret = tape_spacefm(vol->device, 1); + if (ret == 0) { + tape_update_position(vol->device, &final_fm_pos); + n_fm_after++; + } else { + if (ret != -EDEV_EOD_DETECTED) { + goto out; + } + } + } + ret = 0; + switch (n_fm_after){ + case 0: + case 1: + /* Append block does not need to be altered */ + break; + case 2: + /* (index | data | ... eod) - unexpected fm after data, possible incomplete index */ + if (fix) { + ret = final_fm_pos.block-1; + } else { + ret = -LTFS_OP_TO_INV; + } + break; + default: + /* (index | data | ??? | ... eod) - invalid format*/ + ret = -LTFS_OP_TO_INV; + break; + } +out: + return ret; +} + /** * Check a volume for physical consistency. This should be called when there is some doubt about * the validity of the MAM parameters; it reads index files from both partitions and verifies @@ -993,7 +1105,8 @@ int _ltfs_make_lost_found(tape_block_t ip_eod, tape_block_t dp_eod, * * @param fix allow simple fixes to make the tape consistent? * Here, simple means writing an additional logically unmodified copy - * or copies of an index file already present on the tape. + * or copies of an index file already present on the tape. Also + * allows truncating incomplete index at the end of the partition. * @param deep Allow fancy recovery procedures? In particular, this flag enables recovery in the * case where extra blocks (after the last index file on a partition) are found on the * tape. The nature of this recovery is controlled by the recover_extra flag. @@ -1121,8 +1234,28 @@ int ltfs_check_medium(bool fix, bool deep, bool recover_extra, bool recover_syml if (ret < 0) goto out_unlock; - /* Set append position for index partition. */ - if (ip_have_index && ! ip_blocks_after) { + /* Set append position for data partition to end of trailing data. */ + if (dp_have_index && dp_blocks_after) { + ret = _ltfs_find_append_blk_after_idx(vol, vol->label->partid_dp, dp_index->selfptr.block, fix); + if (ret < 0) { + goto out_unlock; + } else { + dp_eod = ret; + check_err(tape_set_append_position(vol->device, dp_num, dp_eod), + 11222E, out_unlock); + } + } + + /* Set append position for index partition to end of trailing data or preceding data */ + if (ip_have_index && ip_blocks_after) { + ret = _ltfs_find_append_blk_after_idx(vol, vol->label->partid_ip, ip_index->selfptr.block, fix); + if (ret <0) { + goto out_unlock; + } else { + ip_eod = ret; + check_err(tape_set_append_position(vol->device, ip_num, ip_eod), + 11222E, out_unlock); + } check_err(tape_set_append_position(vol->device, ip_num, ip_index->selfptr.block - 1), 11222E, out_unlock); } @@ -1190,13 +1323,14 @@ int ltfs_check_medium(bool fix, bool deep, bool recover_extra, bool recover_syml } } /* write to data partition if it doesn't end in an index file */ + ltfs_set_commit_message_reason(SYNC_RECOVERY, vol); if (! dp_have_index || dp_blocks_after) { ltfsmsg(LTFS_INFO, 17259I, "DP", vol->index->selfptr.partition, (unsigned long long)vol->index->selfptr.block); - ret = ltfs_write_index(vol->label->partid_dp, SYNC_RECOVERY, vol); + ret = ltfs_write_index(vol->label->partid_dp, SYNC_RECOVERY, LTFS_FULL_INDEX, vol); } if (!ret) { ltfsmsg(LTFS_INFO, 17259I, "IP", vol->index->selfptr.partition, (unsigned long long)vol->index->selfptr.block); - ltfs_write_index(vol->label->partid_ip, SYNC_RECOVERY, vol); + ltfs_write_index(vol->label->partid_ip, SYNC_RECOVERY, LTFS_FULL_INDEX, vol); } } else { ltfsmsg(LTFS_ERR, 11231E); @@ -1274,17 +1408,20 @@ int ltfs_write_index_conditional(char partition, struct ltfs_volume *vol) CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); - if (partition == ltfs_ip_id(vol) && ! vol->ip_index_file_end) - ret = ltfs_write_index(partition, SYNC_CASCHE_PRESSURE, vol); - else if (partition == ltfs_dp_id(vol) && + if (partition == ltfs_ip_id(vol) && ! vol->ip_index_file_end) { + ltfs_set_commit_message_reason(SYNC_CASCHE_PRESSURE, vol); + ret = ltfs_write_index(partition, SYNC_CASCHE_PRESSURE, LTFS_FULL_INDEX, vol); + } else if (partition == ltfs_dp_id(vol) && (! vol->dp_index_file_end || - (vol->ip_index_file_end && vol->index->selfptr.partition == ltfs_ip_id(vol)))) - ret = ltfs_write_index(partition, SYNC_CASCHE_PRESSURE, vol); + (vol->ip_index_file_end && vol->index->selfptr.partition == ltfs_ip_id(vol)))) { + ltfs_set_commit_message_reason(SYNC_CASCHE_PRESSURE, vol); + ret = ltfs_write_index(partition, SYNC_CASCHE_PRESSURE, LTFS_FULL_INDEX, vol); + } return ret; } -int ltfs_split_symlink( struct ltfs_volume *vol ) +int ltfs_split_symlink(struct ltfs_volume *vol) { size_t i, size; struct dentry *d, *workd; @@ -1395,3 +1532,22 @@ int ltfs_split_symlink( struct ltfs_volume *vol ) free(path); return ret; } + +int ltfs_set_dentry_dirty(struct dentry *d, struct ltfs_volume *vol) +{ + int ret = 0; + char *full_path = NULL; + + if (!d->dirty) { + ret = ltfs_build_fullpath(&full_path, d); + if (!ret) { + incj_modify(full_path, d, vol); + } else { + vol->journal_err = true; + } + } + + d->dirty = true; + + return ret; +} diff --git a/src/libltfs/ltfs_internal.h b/src/libltfs/ltfs_internal.h index 2ce075c3..714aea5e 100644 --- a/src/libltfs/ltfs_internal.h +++ b/src/libltfs/ltfs_internal.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -44,6 +44,10 @@ ** IBM Almaden Research Center ** bbiskebo@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -70,7 +74,8 @@ int ltfs_check_medium(bool fix, bool deep, bool recover_extra, bool recover_syml int ltfs_read_labels(bool trial, struct ltfs_volume *vol); int ltfs_read_one_label(tape_partition_t partition, struct ltfs_label *label, struct ltfs_volume *vol); -int ltfs_read_index(uint64_t eod_pos, bool recover_symlink, struct ltfs_volume *vol); +int ltfs_read_index(uint64_t eod_pos, bool recover_symlink, bool skip_dir, struct ltfs_volume *vol); +int ltfs_read_indexfile(char* filename, bool recover_symlink, struct ltfs_volume *vol); int ltfs_update_cart_coherency(struct ltfs_volume *vol); int ltfs_write_index_conditional(char partition, struct ltfs_volume *vol); @@ -80,6 +85,7 @@ int ltfs_seek_index(char partition, tape_block_t *eod_pos, tape_block_t *index_e bool *fm_after, bool *blocks_after, bool recover_symlink, struct ltfs_volume *vol); void _ltfs_last_ref(struct dentry *d, tape_block_t *dp_last, tape_block_t *ip_last, struct ltfs_volume *vol); -int ltfs_split_symlink( struct ltfs_volume *vol ); +int ltfs_split_symlink(struct ltfs_volume *vol); +int ltfs_set_dentry_dirty(struct dentry *d, struct ltfs_volume *vol); #endif /* __ltfs_internal_h__ */ diff --git a/src/libltfs/ltfstrace.c b/src/libltfs/ltfstrace.c index b02dd239..86567bfb 100644 --- a/src/libltfs/ltfstrace.c +++ b/src/libltfs/ltfstrace.c @@ -97,7 +97,7 @@ * Definition for LTFS trace header information */ #define LTFS_TRACE_SIGNATURE "LTFS_TRC" -#pragma pack(1) +#pragma pack(push, 1) struct trace_header { char signature[8]; /**< Signature for LTFS trace */ uint32_t header_size; /**< Size of trace header */ @@ -108,12 +108,12 @@ struct trace_header { uint32_t trace_size; /**< Whole size of trace (all headers and bodies) */ uint32_t crc; /**< CRC (reserved for future use) */ }; -#pragma pack(0) +#pragma pack(pop) /* * Definitions for LTFS Request header information */ -#pragma pack(1) +#pragma pack(push, 1) struct request_header { uint32_t header_size; /**< Size of request header */ uint32_t num_of_req_trace; /**< Number of request trace descriptrs (always 1) */ @@ -123,12 +123,12 @@ struct request_header { } req_t_desc; /**< Request header descriptor */ uint32_t crc; /**< CRC (reserved for future use) */ }; -#pragma pack(0) +#pragma pack(pop) /* * Definitions for LTFS function trace header */ -#pragma pack(1) +#pragma pack(push, 1) struct function_trace_header { uint32_t header_size; /**< Size of function trace header */ uint32_t num_of_fn_trace; /**< Number of function trace */ @@ -139,19 +139,19 @@ struct function_trace_header { } *req_t_desc; uint32_t crc; /**< CRC (reserved for future use) */ }; -#pragma pack(0) +#pragma pack(pop) /* * Definitions for LTFS function trace data */ -#pragma pack(1) +#pragma pack(push, 1) struct function_entry { uint64_t time; uint64_t function; uint64_t info1; uint64_t info2; }; -#pragma pack(0) +#pragma pack(pop) #define FN_TRACE_ENTRY_SIZE (sizeof(struct function_entry)) @@ -363,8 +363,8 @@ void ltfs_admin_function_trace_completed(uint32_t tid) static void ltfs_function_trace_destroy(void) { if (fs_tr_list) { - struct filesystem_trace_list *fsitem; - for (fsitem=fs_tr_list; fsitem != NULL; fsitem=fsitem->hh.next) { + struct filesystem_trace_list *fsitem, *tmp; + HASH_ITER(hh, fs_tr_list, fsitem, tmp) { destroy_mrsw(&fsitem->fn_entry->trace_lock); free(fsitem->fn_entry); free(fsitem); @@ -372,8 +372,8 @@ static void ltfs_function_trace_destroy(void) fs_tr_list = NULL; } if (admin_tr_list) { - struct admin_trace_list *aditem; - for (aditem=admin_tr_list; aditem != NULL; aditem=aditem->hh.next) { + struct admin_trace_list *aditem, *tmp; + HASH_ITER(hh, admin_tr_list, aditem, tmp) { destroy_mrsw(&aditem->fn_entry->trace_lock); free(aditem->fn_entry); free(aditem); @@ -381,8 +381,8 @@ static void ltfs_function_trace_destroy(void) admin_tr_list = NULL; } if (acomp) { - struct admin_completed_function_trace *tailq_item; - TAILQ_FOREACH (tailq_item, acomp, list) { + struct admin_completed_function_trace *tailq_item, *tmp; + TAILQ_FOREACH_SAFE(tailq_item, acomp, list, tmp) { destroy_mrsw(&tailq_item->trace_lock); free(tailq_item->fn_entry); free(tailq_item); diff --git a/src/libltfs/ltfstrace.h b/src/libltfs/ltfstrace.h index a44b0047..d336fa40 100644 --- a/src/libltfs/ltfstrace.h +++ b/src/libltfs/ltfstrace.h @@ -175,18 +175,18 @@ int ltfs_request_profiler_stop(void); /* * Definitions for LTFS request trace */ -#pragma pack(1) +#pragma pack(push, 1) struct profiler_entry { uint64_t time; uint32_t req_num; uint32_t tid; }; -#pragma pack(0) +#pragma pack(pop) #define PROF_ENTRY_SIZE (sizeof(struct profiler_entry)) #define REQ_PROF_ENTRY_SIZE PROF_ENTRY_SIZE /* Don't record information fields in profler data */ -#pragma pack(1) +#pragma pack(push, 1) struct request_entry { uint64_t time; uint32_t req_num; @@ -194,7 +194,7 @@ struct request_entry { uint64_t info1; uint64_t info2; }; -#pragma pack(0) +#pragma pack(pop) #define REQ_TRACE_ENTRY_SIZE (sizeof(struct request_entry)) #define REQ_TRACE_SIZE (4 * 1024 * 1024) /* 4MB */ diff --git a/src/libltfs/pathname.c b/src/libltfs/pathname.c index ca29c7b7..de76c0aa 100644 --- a/src/libltfs/pathname.c +++ b/src/libltfs/pathname.c @@ -57,7 +57,7 @@ #include #include #include -#ifdef ICU6x +#ifdef USE_UNORM2 #include #else #include @@ -67,7 +67,7 @@ #include #include #include -#ifdef ICU6x +#ifdef USE_UNORM2 #include #else #include @@ -649,13 +649,13 @@ int _pathname_foldcase_icu(const UChar *src, UChar **dest) } /** - * ICU5x/ICU6x handle: gets a reference to a singleton unorm2 object (ICU6x) or to a constant + * ICU5x/USE_UNORM2 handle: gets a reference to a singleton unorm2 object (USE_UNORM2) or to a constant * value that we can use to differentiate between NFC and NFD modes (ICU5x). Neither should be * freed after use. */ static inline void *_unorm_handle(bool nfc, UErrorCode *err) { -#ifdef ICU6x +#ifdef USE_UNORM2 *err = U_ZERO_ERROR; return (void *) unorm2_getInstance(NULL, "nfc", nfc ? UNORM2_COMPOSE : UNORM2_DECOMPOSE, err); #else @@ -666,7 +666,7 @@ static inline void *_unorm_handle(bool nfc, UErrorCode *err) static inline UNormalizationCheckResult _unorm_quickCheck(void *handle, const UChar *src, UChar **dest, UErrorCode *err) { *err = U_ZERO_ERROR; -#ifdef ICU6x +#ifdef USE_UNORM2 const UNormalizer2 *n2 = (const UNormalizer2 *) handle; return unorm2_quickCheck(n2, src, -1, err); #else @@ -678,7 +678,7 @@ static inline UNormalizationCheckResult _unorm_quickCheck(void *handle, const UC static inline int32_t _unorm_normalize(void *handle, const UChar *src, UChar **dest, int32_t len, UErrorCode *err) { *err = U_ZERO_ERROR; -#ifdef ICU6x +#ifdef USE_UNORM2 const UNormalizer2 *n2 = (const UNormalizer2 *) handle; return unorm2_normalize(n2, src, -1, dest ? *dest : NULL, len, err); #else diff --git a/src/libltfs/periodic_sync.c b/src/libltfs/periodic_sync.c index 13571214..4edca3ef 100644 --- a/src/libltfs/periodic_sync.c +++ b/src/libltfs/periodic_sync.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -43,6 +43,7 @@ ** AUTHOR: Atsushi Abe ** IBM Yamato, Japan ** PISTE@jp.ibm.com +** ************************************************************************************* */ @@ -85,6 +86,12 @@ ltfs_thread_return periodic_sync_thread(void* data) if (! priv->keepalive) break; + if (priv->vol->mount_type == MOUNT_ROLLBACK || + priv->vol->mount_type == MOUNT_ROLLBACK_META) { + /* Never call sync on R/O mount */ + continue; + } + ltfs_request_trace(FUSE_REQ_ENTER(REQ_SYNC), 0, 0); ltfsmsg(LTFS_DEBUG, 17067D, "Sync-by-Time"); @@ -94,7 +101,9 @@ ltfs_thread_return periodic_sync_thread(void* data) ltfsmsg(LTFS_WARN, 17063W, __FUNCTION__); } - ret = ltfs_sync_index(SYNC_PERIODIC, true, priv->vol); + ltfs_set_commit_message_reason(SYNC_PERIODIC, priv->vol); + + ret = ltfs_sync_index(SYNC_PERIODIC, true, LTFS_INDEX_AUTO, priv->vol); if (ret < 0) { ltfsmsg(LTFS_INFO, 11030I, ret); priv->keepalive = false; diff --git a/src/libltfs/plugin.c b/src/libltfs/plugin.c index bcf9f6bd..402a10ed 100644 --- a/src/libltfs/plugin.c +++ b/src/libltfs/plugin.c @@ -82,7 +82,7 @@ int plugin_load(struct libltfs_plugin *pl, const char *type, const char *name, CHECK_ARG_NULL(name, -LTFS_NULL_ARG); CHECK_ARG_NULL(config, -LTFS_NULL_ARG); - pl->lib_handle = NULL; + memset(pl, 0, sizeof(*pl)); lib_path = config_file_get_lib(type, name, config); if (! lib_path) { diff --git a/src/libltfs/tape.c b/src/libltfs/tape.c index f23c9f2a..4de147a9 100644 --- a/src/libltfs/tape.c +++ b/src/libltfs/tape.c @@ -485,7 +485,7 @@ int tape_load_tape(struct device_data *dev, void * const kmi_handle, bool force) } /** - * Unroll operations made during tape_load_tape() + * Unroll operations made during tape_load_tape(). * @param dev device to unload * @return 0 on success or a negative value on error */ @@ -721,11 +721,12 @@ int tape_test_unit_ready(struct device_data *dev) } ret = _tape_test_unit_ready(dev); - if (ret < 0) + if (ret < 0) { ltfsmsg(LTFS_ERR, 12029E, ret); - - dev->previous_exist.tv_sec = ts_now.tv_sec; - dev->previous_exist.tv_nsec = ts_now.tv_nsec; + } else { + dev->previous_exist.tv_sec = ts_now.tv_sec; + dev->previous_exist.tv_nsec = ts_now.tv_nsec; + } return ret; } @@ -851,6 +852,7 @@ int tape_seek_append_position(struct device_data *dev, tape_partition_t prt, boo if (ret < 0) { ltfsmsg(LTFS_ERR, 12033E, ret); dev->write_error = true; + return ret; } @@ -996,7 +998,7 @@ int tape_rewind(struct device_data *dev) */ int tape_seek(struct device_data *dev, struct tc_position *pos) { - int ret; + int ret = 0; CHECK_ARG_NULL(dev, -LTFS_NULL_ARG); CHECK_ARG_NULL(pos, -LTFS_NULL_ARG); @@ -1016,8 +1018,14 @@ int tape_seek(struct device_data *dev, struct tc_position *pos) ltfs_mutex_unlock(&dev->read_only_flag_mutex); } } - else { - ret = 0; + + if (IS_WRITE_PERM(-ret)) { + /* + * LOCATE command must not return a WRITE_PERM related error. + * LOCATE is actually read operation, it doesn't make sense to return a WRITE_PERM at all. + */ + ltfsmsg(LTFS_ERR, 17267E, ret, -LTFS_LOCATE_ERROR); + ret = -LTFS_LOCATE_ERROR; } if (ret == 0 && (dev->position.partition != pos->partition || @@ -1051,6 +1059,14 @@ int tape_seek_eod(struct device_data *dev, tape_partition_t partition) ret = dev->backend->locate(dev->backend_data, seekpos, &dev->position); if (ret < 0) { ltfsmsg(LTFS_ERR, 12039E, ret); + if (IS_WRITE_PERM(-ret)) { + /* + * LOCATE command must not return a WRITE_PERM related error. + * LOCATE is actually read operation, it doesn't make sense to return a WRITE_PERM at all. + */ + ltfsmsg(LTFS_ERR, 17267E, ret, -LTFS_LOCATE_ERROR); + ret = -LTFS_LOCATE_ERROR; + } return ret; } @@ -1106,30 +1122,32 @@ int tape_update_position(struct device_data *dev, struct tc_position *pos) return 0; } -int tape_get_physical_block_position(struct device_data *dev, struct tc_position *pos) +int tape_get_first_untransfered_position(struct device_data *dev, struct tc_position *pos) { int ret; - unsigned int block; CHECK_ARG_NULL(dev, -LTFS_NULL_ARG); CHECK_ARG_NULL(pos, -LTFS_NULL_ARG); + /* Update current position, just in case. Because no penalty here */ ret = dev->backend->readpos(dev->backend_data, &dev->position); if (ret < 0) { ltfsmsg(LTFS_ERR, 17132E); return ret; } - ret = dev->backend->get_block_in_buffer(dev->backend_data, &block); + /* Capture first untransferred position */ + ret = dev->backend->get_next_block_to_xfer(dev->backend_data, pos); if (ret < 0) { ltfsmsg(LTFS_ERR, 17132E); return ret; } - memcpy(pos, &dev->position, sizeof(struct tc_position)); - - ltfsmsg(LTFS_DEBUG, 11335D, (int)pos->block, block); - pos->block -= block; + ltfsmsg(LTFS_INFO, 17292I, + (unsigned long long)dev->position.partition, + (unsigned long long)dev->position.block, + (unsigned long long)pos->partition, + (unsigned long long)pos->block); return 0; } @@ -1443,14 +1461,18 @@ static int tape_update_density(struct device_data *dev, int density_code) /** * Format tape for LTFS (make dual-partition tape) * @param dev device to format - * @param index_part partition number for index partition + * @param index_part partition number for index partition, unformat a tape when UINT32_MAX is specified + * @param hard use the hard format (Destroy all data and make 2 partition medium). It might take a + * longer time than soft format because of the media optimization feature + * (media calibration) in LTO9 drive or later. * @return 0 on success or a negative value on error */ -int tape_format(struct device_data *dev, tape_partition_t index_part, int density_code) +int tape_format(struct device_data *dev, tape_partition_t index_part, int density_code, bool hard) { int ret; unsigned char mp_medium_partition[TC_MP_MEDIUM_PARTITION_SIZE+4]; int page_length = TC_MP_MEDIUM_PARTITION_SIZE; + TC_FORMAT_TYPE format_type = TC_FORMAT_PARTITION; CHECK_ARG_NULL(dev, -LTFS_NULL_ARG); CHECK_ARG_NULL(dev->backend, -LTFS_NULL_ARG); @@ -1489,23 +1511,41 @@ int tape_format(struct device_data *dev, tape_partition_t index_part, int densit /* Set appropriate values to the page and Issue Mode Select */ mp_medium_partition[0] = 0x00; mp_medium_partition[1] = 0x00; - mp_medium_partition[19] = 0x01; - mp_medium_partition[20] = 0x20 | (mp_medium_partition[20] & 0x1F); /* Set FDP=0, SDP=0, IDP=1 ==> User Setting */ - mp_medium_partition[22] = 0x09; /* Set partition unit as gigabytes (10^9) */ - if (index_part == 1) { - mp_medium_partition[24] = 0xFF; /* Set Partition0 Capacity */ + + if (index_part == UINT32_MAX) { + mp_medium_partition[19] = 0x00; /* No additional partition, means one-partitioned */ + mp_medium_partition[20] = 0x20 | (mp_medium_partition[20] & 0x1F); /* Set FDP=0, SDP=0, IDP=1 ==> FIXED DATA PARTITION */ + + mp_medium_partition[22] = 0x08; + + mp_medium_partition[24] = 0xFF; mp_medium_partition[25] = 0xFF; - /* Set Partition1 Capacity to 1GB, This value will round up to minimum partition size in FCR3175-r2 */ - /* This field meaning will be chnaged in FCR3175-r3. In r3 n of "minumim partition size * n" should be specified. */ - /* If set this parameter to 1, we can support both specs. */ - /* In r2, this value is rounded up to minimum partition size. In r3, this value is the correct value.*/ - mp_medium_partition[26] = 0x00; /* Set Partition1 Capacity */ - mp_medium_partition[27] = 1; /* will round up to minimum partition size */ + mp_medium_partition[26] = 0x00; + mp_medium_partition[27] = 0x00; } else { - mp_medium_partition[24] = 0x00; /* Set Partition0 Capacity */ - mp_medium_partition[25] = 1; /* will round up to minimum partition size */ - mp_medium_partition[26] = 0xFF; /* Set Partition1 Capacity */ - mp_medium_partition[27] = 0xFF; + if (index_part == 1) { + mp_medium_partition[19] = 0x01; + mp_medium_partition[20] = 0x20 | (mp_medium_partition[20] & 0x1F); /* Set FDP=0, SDP=0, IDP=1 ==> User Setting */ + mp_medium_partition[22] = 0x09; /* Set partition unit as gigabytes (10^9) */ + + mp_medium_partition[24] = 0xFF; /* Set Partition0 Capacity */ + mp_medium_partition[25] = 0xFF; + /* Set Partition1 Capacity to 1GB, This value will round up to minimum partition size in FCR3175-r2 */ + /* This field meaning will be chnaged in FCR3175-r3. In r3 n of "minumim partition size * n" should be specified. */ + /* If set this parameter to 1, we can support both specs. */ + /* In r2, this value is rounded up to minimum partition size. In r3, this value is the correct value.*/ + mp_medium_partition[26] = 0x00; /* Set Partition1 Capacity */ + mp_medium_partition[27] = 1; /* will round up to minimum partition size */ + } else { + mp_medium_partition[19] = 0x01; + mp_medium_partition[20] = 0x20 | (mp_medium_partition[20] & 0x1F); /* Set FDP=0, SDP=0, IDP=1 ==> User Setting */ + mp_medium_partition[22] = 0x09; /* Set partition unit as gigabytes (10^9) */ + + mp_medium_partition[24] = 0x00; /* Set Partition0 Capacity */ + mp_medium_partition[25] = 1; /* will round up to minimum partition size */ + mp_medium_partition[26] = 0xFF; /* Set Partition1 Capacity */ + mp_medium_partition[27] = 0xFF; + } } if (mp_medium_partition[17] > 0x0A) { @@ -1518,8 +1558,11 @@ int tape_format(struct device_data *dev, tape_partition_t index_part, int densit page_length ); - /* Issue Format Medium (destroy all medium data and make 2-partitition medium) */ - ret = dev->backend->format(dev->backend_data, TC_FORMAT_DEST_PART, NULL, NULL, NULL); + /* Issue Format Medium (Make partitioned medium) */ + if (hard) + format_type = TC_FORMAT_DEST_PART; + + ret = dev->backend->format(dev->backend_data, format_type, NULL, NULL, NULL); if (ret < 0) { ltfsmsg(LTFS_ERR, 12053E, ret); return ret; @@ -1531,11 +1574,29 @@ int tape_format(struct device_data *dev, tape_partition_t index_part, int densit } /** - * Unformat tape (make single partition tape) + * Unformat tape (make single partition tape with hard method) + * LTO9 or later drive might triggers the media optimization feature (media calibration) + * when FORMAT_MEDIUM command is issued. It might take a long time to unformat. * @param dev device to format * @return 0 on success or a negative value on error */ int tape_unformat(struct device_data *dev) +{ + int ret; + + ret = tape_format(dev, UINT32_MAX, 0, false); + + return ret; +} + +/** + * Unformat tape (make single partition tape with hard method) + * LTO9 or later drive might triggers the media optimization feature (media calibration) + * when FORMAT_MEDIUM command is issued. It might take a long time to unformat. + * @param dev device to format + * @return 0 on success or a negative value on error + */ +int tape_unformat_hard(struct device_data *dev) { int ret; struct tc_position bom = { .partition = 0, .block = 0, .filemarks = 0 }; @@ -1547,6 +1608,14 @@ int tape_unformat(struct device_data *dev) ret = dev->backend->locate(dev->backend_data, bom, &dev->position); if (ret < 0) { ltfsmsg(LTFS_ERR, 12054E, ret); + if (IS_WRITE_PERM(-ret)) { + /* + * LOCATE command must not return a WRITE_PERM related error. + * LOCATE is actually read operation, it doesn't make sense to return a WRITE_PERM at all. + */ + ltfsmsg(LTFS_ERR, 17267E, ret, -LTFS_LOCATE_ERROR); + ret = -LTFS_LOCATE_ERROR; + } return ret; } @@ -1558,8 +1627,7 @@ int tape_unformat(struct device_data *dev) } /* Reset partition space flag */ - dev->partition_space[0] = - dev->partition_space[1] = PART_WRITABLE; + dev->partition_space[0] = dev->partition_space[1] = PART_WRITABLE; return 0; } @@ -1796,9 +1864,8 @@ int tape_get_media_pool_info(struct ltfs_volume *vol, char **media_name, char ** CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); - if (vol->t_attr->media_pool) { - len = strlen(vol->t_attr->media_pool); - } + vol->t_attr->media_pool[TC_MAM_MEDIA_POOL_SIZE] = '\0'; /* Add a sentinel */ + len = strlen(vol->t_attr->media_pool); if (len == 0) return -1; @@ -2589,6 +2656,18 @@ int tape_is_cartridge_loadable(struct device_data *dev) return ret; } +int tape_logsense(struct device_data *dev, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size) +{ + int ret = -EDEV_UNKNOWN; + + CHECK_ARG_NULL(dev, -LTFS_NULL_ARG); + + ret = dev->backend->logsense(dev->backend_data, page, subpage, buf, size); + + return ret; +} + /** * Wait the drive goes to ready state * @param device handle to tape device @@ -3438,7 +3517,49 @@ int read_tape_attribute(struct ltfs_volume *vol, char **val, const char *name) } /** - * Evaluate the tape can be mountable + * Generic I/F to get MAM. + * @param dev a pointer to the tape device + * @param part partition to get MAM + * @param buf pointer of the buffer + * @param size length of the buffer + * @return attr length on success or a negative value on error. + */ +int tape_read_attr(struct device_data *dev, const tape_partition_t part, + unsigned char *buf, const size_t size) +{ + int ret; + unsigned char *inner_buf = NULL; + unsigned int len = 0; + + CHECK_ARG_NULL(dev, -LTFS_NULL_ARG); + CHECK_ARG_NULL(dev->backend, -LTFS_NULL_ARG); + + inner_buf = calloc(1, MAXMAM_SIZE); /* Assume max length of MAM is 0xFFFF */ + if (!inner_buf) + return -LTFS_NO_MEMORY; + + ret = dev->backend->read_attribute(dev->backend_data, part, + 0, + inner_buf, + MAXMAM_SIZE); + if (!ret) { + len = ((int)inner_buf[2] << 8) + (int)inner_buf[3] + 4; + + if (size > len) + memcpy(buf, inner_buf, len); + else + memcpy(buf, inner_buf, size); + + ret = len; + } + + free(inner_buf); + + return ret; +} + +/** + * Evaluate the tape can be mountable. * @param device a pointer to the tape device * @param barcode barcode of the tape (NULL is accepted if cart_type and density is available) * @param cart_type cartridge type in CM @@ -3485,7 +3606,7 @@ int tape_is_reformattable(struct device_data *dev, unsigned char cart_type, unsi * Enable profiler function * @param device a pointer to the tape device * @param work_dir work directory to store profiler data - * @paran enable enable or disable profiler function of this backend + * @param enable enable or disable profiler function of this backend * @return 0 on success or a negative value on error */ int tape_set_profiler(struct device_data *dev, char *work_dir, bool enable) @@ -3496,3 +3617,31 @@ int tape_set_profiler(struct device_data *dev, char *work_dir, bool enable) return dev->backend->set_profiler(dev->backend_data, work_dir, enable); } + +/** + * Prepares GRAO parameter list, and sends rao request. + * All data should be set in rao before this is called. + * @param dev a pointer to the tape device + * @param rao a pointer to the rao data + * @return 0 on success or a negative value on error + */ +int tape_rao_request(struct device_data *dev, struct rao_mod *rao) +{ + int ret = 0; + + /* run GRAO */ + ret = dev->backend->grao(dev->backend_data, rao->in_buf, rao->in_size); + if (ret < 0) { + ltfsmsg(LTFS_INFO, 17275I, "GRAO", ret); //GRAO command returns error + return ret; + } + + /* run RRAO */ + ret = dev->backend->rrao(dev->backend_data, rao->out_buf, rao->buf_size, &rao->out_size); + if (ret < 0) { + ltfsmsg(LTFS_INFO, 17275I, "RRAO", ret); + return ret; + } + + return ret; +} diff --git a/src/libltfs/tape.h b/src/libltfs/tape.h index c3131578..ca340130 100644 --- a/src/libltfs/tape.h +++ b/src/libltfs/tape.h @@ -75,6 +75,8 @@ extern "C" { #define MB (KB * 1024) #define GB (MB * 1024) +#define COMMAND_DESCRIPTION_LENGTH (32) + #define NEED_REVAL(ret) (ret == -EDEV_POR_OR_BUS_RESET \ || ret == -EDEV_MEDIUM_MAY_BE_CHANGED \ || ret == -EDEV_RESERVATION_PREEMPTED \ @@ -84,6 +86,14 @@ extern "C" { #define IS_UNEXPECTED_MOVE(ret) (ret == -EDEV_MEDIUM_REMOVAL_REQ) +struct rao_mod { + unsigned char *in_buf; /**< buffer to set in grao */ + unsigned char *out_buf; /**< buffer returned from rrao */ + size_t buf_size; /**< buffer size of in_buf and out_buf */ + uint32_t in_size; /**< Valid length of in_buf */ + size_t out_size; /**< Valid length of out_buf */ +}; + struct device_data { struct tc_position position; /**< Current head position */ tape_block_t append_pos[2]; /**< Append positions, 0 means append at EOD */ @@ -155,7 +165,7 @@ int tape_update_position(struct device_data *dev, struct tc_position *pos); int tape_seek(struct device_data *dev, struct tc_position *pos); int tape_seek_eod(struct device_data *dev, tape_partition_t partition); int tape_seek_append_position(struct device_data *dev, tape_partition_t prt, bool unlock_write); -int tape_get_physical_block_position(struct device_data *dev, struct tc_position *pos); +int tape_get_first_untransfered_position(struct device_data *dev, struct tc_position *pos); int tape_spacefm(struct device_data *dev, int count); ssize_t tape_read(struct device_data *dev, char *buf, size_t count, const bool unusual_size, @@ -163,8 +173,9 @@ ssize_t tape_read(struct device_data *dev, char *buf, size_t count, const bool u int tape_erase(struct device_data *dev, bool long_erase); int tape_reset_capacity(struct device_data *dev); -int tape_format(struct device_data *dev, tape_partition_t index_part, int density_code); +int tape_format(struct device_data *dev, tape_partition_t index_part, int density_code, bool hard); int tape_unformat(struct device_data *dev); +int tape_unformat_hard(struct device_data *dev); ssize_t tape_write(struct device_data *dev, const char *buf, size_t count, bool ignore_less, bool ignore_nospc); int tape_write_filemark(struct device_data *dev, uint8_t count, bool ignore_less, bool ignore_nospc, bool immed); @@ -203,6 +214,8 @@ int tape_enable_append_only_mode(struct device_data *dev, bool enable); int tape_get_append_only_mode_setting(struct device_data *dev, bool *enabled); int tape_is_cartridge_loadable(struct device_data *dev); +int tape_logsense(struct device_data *dev, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size); int tape_wait_device_ready(struct device_data *dev, void * const kmi_handle); int tape_set_key(struct device_data *dev, const unsigned char *keyalias, const unsigned char *key); int tape_clear_key(struct device_data *device, void * const kmi_handle); @@ -220,10 +233,13 @@ int tape_get_attribute_from_cm(struct device_data *dev, struct tape_attr *t_attr void tape_load_all_attribute_from_cm(struct device_data *dev, struct tape_attr *t_attr); int update_tape_attribute (struct ltfs_volume *vol, const char *new_value, int type, int size); int read_tape_attribute (struct ltfs_volume *vol, char **val, const char *name); +int tape_read_attr(struct device_data *dev, const tape_partition_t part, + unsigned char *buf, const size_t size); int tape_is_mountable(struct device_data *dev, char *barcode, unsigned char cart_type, unsigned char density); int tape_is_reformattable(struct device_data *dev, unsigned char cart_type, unsigned char density); int tape_set_profiler(struct device_data *dev, char *work_dir, bool enable); +int tape_rao_request(struct device_data *dev, struct rao_mod *rao); static inline char* tape_get_serialnumber(struct device_data *dev) { diff --git a/src/libltfs/tape_ops.h b/src/libltfs/tape_ops.h index 80b7e801..deae04a2 100644 --- a/src/libltfs/tape_ops.h +++ b/src/libltfs/tape_ops.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2023 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -166,6 +166,7 @@ enum { TC_DC_JAG5 = 0x55, TC_DC_JAG5A = 0x56, TC_DC_JAG6 = 0x57, + TC_DC_JAG7 = 0x59, TC_DC_JAG1E = 0x71, TC_DC_JAG2E = 0x72, TC_DC_JAG3E = 0x73, @@ -173,6 +174,7 @@ enum { TC_DC_JAG5E = 0x75, TC_DC_JAG5AE = 0x76, TC_DC_JAG6E = 0x77, + TC_DC_JAG7E = 0x79, }; #define ALL_MEDIA_DENSITY 0 @@ -581,6 +583,7 @@ struct tape_ops { /** * Get capacity data from a device. + * * @param device Device handle returned by the backend's open(). * @param cap On success, the backend must fill this structure with the total and remaining * capacity values of the two partitions on the medium, in units of 1048576 bytes. @@ -590,16 +593,17 @@ struct tape_ops { /** * Send a SCSI Log Sense command to a device. - * libltfs does not currently use this function, but it may be useful internally (for example, - * ibmtape uses it from its remaining_capacity() function). + * * @param device Device handle returned by the backend's open(). * @param page Log page to query. + * @param subpage Specify the sub page of the log page to query. * @param buf On success, the backend must fill this buffer with the log page's value. * @param size Buffer size. - * @return 0 on success or a negative value on error. Backends for which Log Sense is + * @return Page length on success or a negative value on error. Backends for which Log Sense is * meaningless should return -1. */ - int (*logsense)(void *device, const uint8_t page, unsigned char *buf, const size_t size); + int (*logsense)(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size); /** * Send a SCSI Mode Sense(10) command to a device. @@ -664,6 +668,8 @@ struct tape_ops { * Read a MAM parameter from a device. * For performance reasons, it is recommended that all backends implement MAM parameter support. * However, this support is technically optional. + * Normally the buffer doesn't include header of contents. But it includes only when the size is + * MAXMAM_SIZE. * @param device Device handle returned by the backend's open(). * @param part Partition to read the parameter from. * @param id Attribute ID to read. libltfs uses TC_MAM_PAGE_VCR and TC_MAM_PAGE_COHERENCY. @@ -911,18 +917,39 @@ struct tape_ops { int (*set_profiler)(void *device, char *work_dir, bool enable); /** - * Get block number stored in the drive buffer + * Get first block number which is not transferred to the medium yet in the buffer * @param device A pointer to the tape device - * @param block Number of blocks stored in the drive buffer + * @param pos Position of the record that is not transferred yet in the buffer * @return 0 on success or a negative value on error */ - int (*get_block_in_buffer)(void *device, unsigned int *block); + int (*get_next_block_to_xfer)(void *device, struct tc_position *pos); /** * Check if the generation of tape drive and the current loaded cartridge is read-only combination * @param device Device handle returned by the backend's open(). */ bool (*is_readonly)(void *device); + + /** + * Submit the GRAO (Generate Recommended Access Order) command. + * The generated result can be called from the RRAO command. + * @param device A pointer to the tape device + * @param buf buffer data to process + * @param len Length of the buffer + * @return 0 on success or a negative value on error + */ + int (*grao)(void *device, unsigned char *buf, const uint32_t len); + + /** + * Submit the RRAO (Receive Recommended Access Order) command. + * This should be called after the GRAO command. + * @param device A pointer to the tape device + * @param[out] buf rrao, rao result in binary. You need to parse it by yourself. + * @param len Length of the buffer + * @param[out] out_size size of valid data returned from the device + * @return 0 on success or a negative value on error + */ + int (*rrao)(void *device, unsigned char *buf, const uint32_t len, size_t *out_size); }; /** diff --git a/src/libltfs/xattr.c b/src/libltfs/xattr.c index a4cacc1c..e19d9ff3 100644 --- a/src/libltfs/xattr.c +++ b/src/libltfs/xattr.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -62,653 +66,335 @@ #include "pathname.h" #include "tape.h" #include "ltfs_internal.h" +#include "inc_journal.h" #include "arch/time_internal.h" -int _xattr_seek(struct xattr_info **out, struct dentry *d, const char *name); -int _xattr_lock_dentry(const char *name, bool modify, struct dentry *d, struct ltfs_volume *vol); -void _xattr_unlock_dentry(const char *name, bool modify, struct dentry *d, struct ltfs_volume *vol); -const char *_xattr_strip_name(const char *name); -int _xattr_list_physicals(struct dentry *d, char *list, size_t size); -bool _xattr_is_virtual(struct dentry *d, const char *name, struct ltfs_volume *vol); -int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char *name, - struct ltfs_volume *vol); -int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, - size_t size, struct ltfs_volume *vol); -int _xattr_remove_virtual(struct dentry *d, const char *name, struct ltfs_volume *vol); -bool _xattr_is_worm_ea(const char *name); - /* Helper functions for formatting virtual EA output */ -int _xattr_get_cartridge_health(cartridge_health_info *h, int64_t *val, char **outval, - const char *msg, struct ltfs_volume *vol); -int _xattr_get_cartridge_health_u64(cartridge_health_info *h, uint64_t *val, char **outval, - const char *msg, struct ltfs_volume *vol); -int _xattr_get_cartridge_capacity(struct device_capacity *cap, unsigned long *val, char **outval, - const char *msg, struct ltfs_volume *vol); -int _xattr_get_time(struct ltfs_timespec *val, char **outval, const char *msg); -int _xattr_get_dentry_time(struct dentry *d, struct ltfs_timespec *val, char **outval, - const char *msg); -int _xattr_get_string(const char *val, char **outval, const char *msg); -int _xattr_get_tapepos(struct tape_offset *val, char **outval, const char *msg); -int _xattr_get_partmap(struct ltfs_label *label, char **outval, const char *msg); -int _xattr_get_version(int version, char **outval, const char *msg); - -int _xattr_set_time(struct dentry *d, struct ltfs_timespec *out, const char *value, size_t size, - const char *msg, struct ltfs_volume *vol); - -int _xattr_get_vendorunique_xattr(char **outval, const char *msg, struct ltfs_volume *vol); -int _xattr_set_vendorunique_xattr(const char *name, const char *value, size_t size, struct ltfs_volume *vol); - -int xattr_do_set(struct dentry *d, const char *name, const char *value, size_t size, - struct xattr_info *xattr) +static int _xattr_get_cartridge_health(cartridge_health_info *h, int64_t *val, char **outval, + const char *msg, struct ltfs_volume *vol) { - int ret = 0; - - /* clear existing xattr or set up new one */ - if (xattr) { - if (xattr->value) { - free(xattr->value); - xattr->value = NULL; - } - } else { - xattr = (struct xattr_info *) calloc(1, sizeof(struct xattr_info)); - if (! xattr) { - ltfsmsg(LTFS_ERR, 10001E, "xattr_do_set: xattr"); + int ret = ltfs_get_cartridge_health(h, vol); + if (ret == 0) { + ret = asprintf(outval, "%"PRId64, *val); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, msg); + *outval = NULL; return -LTFS_NO_MEMORY; } - xattr->key.name = strdup(name); - if (! xattr->key.name) { - ltfsmsg(LTFS_ERR, 10001E, "xattr_do_set: xattr key"); + } else + *outval = NULL; + return ret; +} + +static int _xattr_get_cartridge_health_u64(cartridge_health_info *h, uint64_t *val, char **outval, + const char *msg, struct ltfs_volume *vol) +{ + int ret = ltfs_get_cartridge_health(h, vol); + if (ret == 0 && (int64_t)(*val) != UNSUPPORTED_CARTRIDGE_HEALTH) { + ret = asprintf(outval, "%"PRIu64, *val); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, msg); + *outval = NULL; ret = -LTFS_NO_MEMORY; - goto out_free; } - xattr->key.percent_encode = fs_is_percent_encode_required(xattr->key.name); - TAILQ_INSERT_HEAD(&d->xattrlist, xattr, list); - } - - /* copy new value */ - xattr->size = size; - if (size > 0) { - xattr->value = (char *)malloc(size); - if (! xattr->value) { - ltfsmsg(LTFS_ERR, 10001E, "xattr_do_set: xattr value"); + } else if (ret == 0) { + ret = asprintf(outval, "%"PRId64, UNSUPPORTED_CARTRIDGE_HEALTH); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, msg); + *outval = NULL; ret = -LTFS_NO_MEMORY; - goto out_remove; } - memcpy(xattr->value, value, size); - } - return 0; + } else + *outval = NULL; + return ret; +} -out_remove: - TAILQ_REMOVE(&d->xattrlist, xattr, list); -out_free: - if (xattr->key.name) - free(xattr->key.name); - free(xattr); +static int _xattr_get_cartridge_capacity(struct device_capacity *cap, unsigned long *val, + char **outval, const char *msg, struct ltfs_volume *vol) +{ + double scale = vol->label->blocksize / 1048576.0; + int ret = ltfs_capacity_data_unlocked(cap, vol); + if (ret == 0) { + ret = asprintf(outval, "%lu", (unsigned long)((*val) * scale)); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, msg); + *outval = NULL; + return -LTFS_NO_MEMORY; + } + } else + *outval = NULL; return ret; } -/** - * Set an extended attribute. - * @param d File or directory to set the xattr on. - * @param name Name to set. - * @param value Value to set, may be binary, not necessarily null-terminated. - * @param size Size of value in bytes. - * @param flags XATTR_REPLACE to fail if xattr doesn't exist, XATTR_CREATE to fail if it does - * exist, or 0 to ignore any existing value. - * @return 0 on success or a negative value on error. - */ -int xattr_set(struct dentry *d, const char *name, const char *value, size_t size, - int flags, struct ltfs_volume *vol) +static int _xattr_get_time(struct ltfs_timespec *val, char **outval, const char *msg) { - struct xattr_info *xattr; - bool replace, create; int ret; - bool is_worm_cart = false; - bool disable_worm_ea = false; - char *new_value="1"; - bool write_idx = false; - - CHECK_ARG_NULL(d, -LTFS_NULL_ARG); - CHECK_ARG_NULL(name, -LTFS_NULL_ARG); - CHECK_ARG_NULL(value, -LTFS_NULL_ARG); - CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); - if (size > LTFS_MAX_XATTR_SIZE) - return -LTFS_LARGE_XATTR; /* this is the error returned by ext3 when the xattr is too large */ - - replace = flags & XATTR_REPLACE; - create = flags & XATTR_CREATE; - - ret = _xattr_lock_dentry(name, true, d, vol); - if (ret < 0) - return ret; - ret = tape_get_worm_status(vol->device, &is_worm_cart); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 17237E, "set xattr: cart stat"); - ret = -LTFS_XATTR_ERR; - goto out_unlock; + ret = xml_format_time(*val, outval); + if (! (*outval)) { + ltfsmsg(LTFS_ERR, 11145E, msg); + return -LTFS_NO_MEMORY; } - if ((is_worm_cart && (d->is_immutable || (d->is_appendonly && strcmp(name, "ltfs.vendor.IBM.immutable")))) - || (!is_worm_cart && (d->is_immutable || d->is_appendonly) && !_xattr_is_worm_ea(name))) { - /* EA cannot be set in case of immutable/appendonly */ - ltfsmsg(LTFS_ERR, 17237E, "set xattr: WORM entry"); - ret = -LTFS_RDONLY_XATTR; - goto out_unlock; - } + return ret; +} - /* Check if this is a user-writeable virtual xattr */ - if (_xattr_is_virtual(d, name, vol)) { - ret = _xattr_set_virtual(d, name, value, size, vol); - if (ret == -LTFS_NO_XATTR) - ret = -LTFS_RDONLY_XATTR; - goto out_unlock; - } +static int _xattr_get_dentry_time(struct dentry *d, struct ltfs_timespec *val, char **outval, + const char *msg) +{ + int ret; + acquireread_mrsw(&d->meta_lock); + ret = _xattr_get_time(val, outval, msg); + releaseread_mrsw(&d->meta_lock); + return ret; +} - /* In the future, there could be user-writeable reserved xattrs. For now, just deny - * writes to all reserved xattrs not covered by the user-writeable virtual xattrs above. */ - if (strcasestr(name, "ltfs") == name && strcmp(name, "ltfs.spannedFileOffset") && strcmp(name, "ltfs.mediaPool.name") && - strcasestr(name, "ltfs.permissions.") != name && !_xattr_is_worm_ea(name)) { - ret = -LTFS_RDONLY_XATTR; - goto out_unlock; +static int _xattr_get_tapepos(struct tape_offset *val, char **outval, const char *msg) +{ + int ret = asprintf(outval, "%c:%"PRIu64, val->partition, val->block); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, msg); + return -LTFS_NO_MEMORY; } + return 0; +} - acquirewrite_mrsw(&d->meta_lock); - - /* Search for existing xattr with this name. */ - ret = _xattr_seek(&xattr, d, name); +static int _xattr_get_partmap(struct ltfs_label *label, char **outval, const char *msg) +{ + int ret = asprintf(outval, "I:%c,D:%c", label->partid_ip, label->partid_dp); if (ret < 0) { - ltfsmsg(LTFS_ERR, 11122E, ret); - releasewrite_mrsw(&d->meta_lock); - goto out_unlock; - } - if (create && xattr) { - releasewrite_mrsw(&d->meta_lock); - ret = -LTFS_XATTR_EXISTS; - goto out_unlock; - } else if (replace && ! xattr) { - releasewrite_mrsw(&d->meta_lock); - ret = -LTFS_NO_XATTR; - goto out_unlock; + ltfsmsg(LTFS_ERR, 10001E, msg); + return -LTFS_NO_MEMORY; } - if (_xattr_is_worm_ea(name)) { - disable_worm_ea = (strncmp(value, "0", size) == 0); + return 0; +} - if (is_worm_cart && disable_worm_ea) { - ltfsmsg(LTFS_ERR, 17237E, "set xattr: clear WORM"); - releasewrite_mrsw(&d->meta_lock); - ret = -LTFS_XATTR_ERR; - goto out_unlock; - } - if (!disable_worm_ea) { - /* All values other than 0 is treated as 1 */ - value = new_value; - size = strlen(new_value); +static int _xattr_get_version(int version, char **outval, const char *msg) +{ + int ret; + if (version == 10000) { + *outval = strdup("1.0"); + if (! (*outval)) { + ltfsmsg(LTFS_ERR, 10001E, msg); + return -LTFS_NO_MEMORY; } - } - - if (!strcmp(name, "ltfs.mediaPool.name")) { - ret = tape_set_media_pool_info(vol, value, size, true); + } else { + ret = asprintf(outval, "%d.%d.%d", version/10000, (version % 10000)/100, version % 100); if (ret < 0) { - releasewrite_mrsw(&d->meta_lock); - goto out_unlock; + ltfsmsg(LTFS_ERR, 10001E, msg); + return -LTFS_NO_MEMORY; } - write_idx = true; } + return 0; +} - /* Set extended attribute */ - ret = xattr_do_set(d, name, value, size, xattr); - if (ret < 0) { - releasewrite_mrsw(&d->meta_lock); - goto out_unlock; - } +static int _xattr_set_time(struct dentry *d, struct ltfs_timespec *out, const char *value, + size_t size, const char *msg, struct ltfs_volume *vol) +{ + int ret; + struct ltfs_timespec t; + char *value_null_terminated; - /* update metadata */ - if (!strcmp(name, "ltfs.vendor.IBM.immutable")) { - d->is_immutable = !disable_worm_ea; - ltfsmsg(LTFS_INFO, 17238I, "immutable", d->is_immutable, d->name.name); - } - else if (!strcmp(name, "ltfs.vendor.IBM.appendonly")) { - d->is_appendonly = !disable_worm_ea; - ltfsmsg(LTFS_INFO, 17238I, "appendonly", d->is_appendonly, d->name.name); + value_null_terminated = malloc(size + 1); + if (! value_null_terminated) { + ltfsmsg(LTFS_ERR, 10001E, msg); + return -LTFS_NO_MEMORY; } + memcpy(value_null_terminated, value, size); + value_null_terminated[size] = '\0'; + + ret = xml_parse_time(false, value_null_terminated, &t); + free(value_null_terminated); + if (ret < 0) + return -LTFS_BAD_ARG; + + acquirewrite_mrsw(&d->meta_lock); + *out = t; + + ltfs_set_dentry_dirty(d, vol); - get_current_timespec(&d->change_time); releasewrite_mrsw(&d->meta_lock); - d->dirty = true; + ltfs_set_index_dirty(true, false, vol->index); + return ret; +} - if (write_idx) - ret = ltfs_sync_index(SYNC_EA, false, vol); - else - ret = 0; +static int _xattr_get_vendorunique_xattr(char **outval, const char *msg, struct ltfs_volume *vol) +{ + int ret; + + ret = ltfs_get_vendorunique_xattr(msg, outval, vol); + if (ret != 0) + *outval = NULL; -out_unlock: - _xattr_unlock_dentry(name, true, d, vol); return ret; } -/** - * Get an extended attribute. Returns an error if the provided buffer is not large enough - * to contain the attribute value. - * @param d File/directory to check - * @param name Xattr name - * @param value On success, contains xattr value - * @param size Output buffer size in bytes - * @param vol LTFS volume - * @return if size is nonzero, number of bytes returned in the value buffer. if size is zero, - * number of bytes in the xattr value. returns a negative value on error. If the - * operation needs to be restarted, then -LTFS_RESTART_OPERATION is returned instead. - */ -int xattr_get(struct dentry *d, const char *name, char *value, size_t size, - struct ltfs_volume *vol) +static int _xattr_set_vendorunique_xattr(const char *name, const char *value, size_t size, + struct ltfs_volume *vol) { - struct xattr_info *xattr = NULL; int ret; - CHECK_ARG_NULL(d, -LTFS_NULL_ARG); - CHECK_ARG_NULL(name, -LTFS_NULL_ARG); - CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); - if (size > 0 && ! value) { - ltfsmsg(LTFS_ERR, 11123E); - return -LTFS_BAD_ARG; - } - - ret = _xattr_lock_dentry(name, false, d, vol); - if (ret < 0) - return ret; - - /* Try to get a virtual xattr first. */ - if (_xattr_is_virtual(d, name, vol)) { - ret = _xattr_get_virtual(d, value, size, name, vol); - if (ret == -LTFS_DEVICE_FENCED) { - _xattr_unlock_dentry(name, false, d, vol); - ret = ltfs_wait_revalidation(vol); - return (ret == 0) ? -LTFS_RESTART_OPERATION : ret; - } else if (NEED_REVAL(ret)) { - _xattr_unlock_dentry(name, false, d, vol); - ret = ltfs_revalidate(false, vol); - return (ret == 0) ? -LTFS_RESTART_OPERATION : ret; - } else if (IS_UNEXPECTED_MOVE(ret)) { - vol->reval = -LTFS_REVAL_FAILED; - _xattr_unlock_dentry(name, false, d, vol); - return ret; - }else if (ret != -LTFS_NO_XATTR) { - /* if ltfs.sync is specified, don't print any message */ - if (ret < 0 && ret != -LTFS_RDONLY_XATTR) - ltfsmsg(LTFS_ERR, 11128E, ret); - goto out_unlock; - } - } + ret = ltfs_set_vendorunique_xattr(name, value, size, vol); - acquireread_mrsw(&d->meta_lock); + return ret; +} - /* Look for a real xattr. */ - ret = _xattr_seek(&xattr, d, name); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 11129E, ret); - releaseread_mrsw(&d->meta_lock); - goto out_unlock; +/** + * Check xattr name is WORM related one or not + * + * @param name EA name to check + * @return true if name is related WORM. Otherwise false. + */ +static inline bool _xattr_is_worm_ea(const char *name) +{ + if (!strcmp(name, "ltfs.vendor.IBM.immutable") || !strcmp(name, "ltfs.vendor.IBM.appendonly")) { + /* WORM related xattr */ + return true; } + return false; +} - /* Generate output. */ - ret = 0; - if (! xattr) { - /* There's no such extended attribute */ - ret = -LTFS_NO_XATTR; - } else if (size && xattr->size > size) { - /* There is no space to fill the buffer */ - ret = -LTFS_SMALL_BUFFER; - } else if (size) { - /* Copy the extended attribute to the requester */ - memcpy(value, xattr->value, xattr->size); - ret = xattr->size; - } else /* size is zero */ { - /* Return how many bytes will be necessary to read this xattr */ - ret = xattr->size; +/** + * Check xattr name is stored VEA or not + * + * Extended attribute starts from 'ltfs.' is reserved and treated as Virtual Extended Attribute (VEA) + * and they aren't stored into index. But 'ltfs.permissions.*' and 'ltfs.hash.*' is introduced from + * LTFS Format Spec 2.4 to store values into index. + * This function checks provided name shall be stored VEA or not. + * + * @param name EA name to check + * @return true if name is stored VEA. Otherwise false. + */ +static inline bool _xattr_is_stored_vea(const char *name) +{ + if (strcmp(name, "ltfs.spannedFileOffset") && + strcasestr(name, "ltfs.permissions.") != name && + strcasestr(name, "ltfs.hash.") != name) + { + return false; } - releaseread_mrsw(&d->meta_lock); - -out_unlock: - _xattr_unlock_dentry(name, false, d, vol); - return ret; + return true; } +/* Local functions */ + /** - * Copy a list of extended attribute names to a user-provided buffer. - * @param d File/directory to get the list of extended attributes from - * @param list Output buffer for xattr names - * @param size Output buffer size in bytes - * @param vol LTFS volume - * @return number of bytes in buffer on success, or a negative value on error. + * Search for an xattr with the given name. Must call this function with a lock on + * the dentry's meta_lock. + * @param out On success, points to the xattr that was found. + * @param d Dentry to search. + * @param name Name to search for. + * @return 1 if xattr was found, 0 if not, or a negative value on error. */ -int xattr_list(struct dentry *d, char *list, size_t size, struct ltfs_volume *vol) +static int _xattr_seek(struct xattr_info **out, struct dentry *d, const char *name) { - int ret, nbytes = 0; + struct xattr_info *entry; - CHECK_ARG_NULL(d, -LTFS_NULL_ARG); - CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); - if (size > 0 && ! list) { - ltfsmsg(LTFS_ERR, 11130E); - return -LTFS_BAD_ARG; + *out = NULL; + TAILQ_FOREACH(entry, &d->xattrlist, list) { + if (! strcmp(entry->key.name, name)) { + *out = entry; + break; + } } - acquireread_mrsw(&d->meta_lock); - - /* Fill the buffer with only real xattrs. */ - if (size) - memset(list, 0, size); + if (*out) + return 1; + else + return 0; +} - ret = _xattr_list_physicals(d, list, size); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 11133E, ret); - goto out; +/** + * Take the volume lock and dentry contents_lock as appropriate for the EA and the access type. + * @param name EA name being read or written. + * @param modify True if EA will be set or deleted, false if it will be read. + * @param d Dentry under consideration. + * @param vol LTFS volume. + * @return 0 on success or -LTFS_REVAL_FAILED if the medium is invalid. + */ +static int _xattr_lock_dentry(const char *name, bool modify, struct dentry *d, struct ltfs_volume *vol) +{ + /* EAs that read the extent list need to take the contents_lock */ + if (! strcmp(name, "ltfs.startblock") + || ! strcmp(name, "ltfs.partition")) { + acquireread_mrsw(&d->contents_lock); } - nbytes += ret; - - /* - * There used to be an _xattr_list_virtuals function which was called here. - * Listing virtual xattrs causes problems with files copied from LTFS to another filesystem - * which are attempted to be brought back. Since the copy utility may also copy the - * reserved virtual extended attributes, the copy operation will fail with permission - * denied problems. - */ - - /* Was the buffer large enough? */ - if (size && (size_t)nbytes > size) - ret = -LTFS_SMALL_BUFFER; -out: - releaseread_mrsw(&d->meta_lock); - if (ret < 0) - return ret; - return nbytes; + /* Other EAs either need no additional locks, or they need the meta_lock. + * The caller is responsible for taking the meta_lock as necessary. */ + return 0; } /** - * Actually remove an extended attribute. - * @param dentry dentry to operate on - * @param name xattr name to delete - * @param force true to force removal, false to verify namespaces first - * @param vol LTFS volume - * @return 0 on success or a negative value on error + * Undo locking performed in _xattr_lock_dentry. + * @param name EA name being read or written. + * @param modify True if EA was set or deleted, false if it was read. + * @param d Dentry under consideration. + * @param vol LTFS volume. */ -int xattr_do_remove(struct dentry *d, const char *name, bool force, struct ltfs_volume *vol) +static void _xattr_unlock_dentry(const char *name, bool modify, struct dentry *d, struct ltfs_volume *vol) { - int ret; - struct xattr_info *xattr; + /* EAs that read the extent list need to take the contents_lock */ + if (! strcmp(name, "ltfs.startblock") + || ! strcmp(name, "ltfs.partition")) { + releaseread_mrsw(&d->contents_lock); + } +} - acquirewrite_mrsw(&d->meta_lock); +/** + * List real extended attributes for a dentry. Must be called with a read lock held on + * the dentry's meta_lock. + * @param d Dentry to list. + * @param list Output buffer. + * @param size Output buffer size, may be 0. + * @return Number of bytes in listed extended attributes, or a negative value on error. + */ +static int _xattr_list_physicals(struct dentry *d, char *list, size_t size) +{ + struct xattr_info *entry; + char *prefix = "\0", *new_name; + int prefixlen = 0, namelen; + int ret = 0, nbytes = 0; - /* Look for a real extended attribute. */ - ret = _xattr_seek(&xattr, d, name); +#if ((!defined (__APPLE__)) && (!defined (mingw_PLATFORM))) + ret = pathname_unformat("user.", &prefix); if (ret < 0) { - ltfsmsg(LTFS_ERR, 11140E, ret); - releasewrite_mrsw(&d->meta_lock); + ltfsmsg(LTFS_ERR, 11141E, ret); return ret; - } else if (! xattr) { - releasewrite_mrsw(&d->meta_lock); - return -LTFS_NO_XATTR; } + prefixlen = strlen(prefix); +#endif /* (!defined (__APPLE__)) && (!defined (mingw_PLATFORM)) */ - if (! force) { - /* If this xattr is in the reserved namespace, the user can't remove it. */ - /* TODO: in the future, there could be user-removable reserved xattrs. */ - if (strcasestr(name, "ltfs") == name && strcmp(name, "ltfs.spannedFileOffset") && - strcasestr(name, "ltfs.permissions.") != name && !_xattr_is_worm_ea(name) ) { - releasewrite_mrsw(&d->meta_lock); - return -LTFS_RDONLY_XATTR; + TAILQ_FOREACH(entry, &d->xattrlist, list) { + ret = pathname_unformat(entry->key.name, &new_name); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 11142E, ret); + goto out; } - } - /* Remove the xattr. */ - TAILQ_REMOVE(&d->xattrlist, xattr, list); - get_current_timespec(&d->change_time); - releasewrite_mrsw(&d->meta_lock); + if(strncmp(new_name, LTFS_LIVELINK_EA_NAME, strlen(LTFS_LIVELINK_EA_NAME) + 1)) { + namelen = strlen(new_name); - free(xattr->key.name); - if (xattr->value) - free(xattr->value); - free(xattr); + nbytes += prefixlen + namelen + 1; + if (size && (size_t)nbytes <= size) { + memcpy(list, prefix, prefixlen); + list += prefixlen; + memcpy(list, new_name, namelen); + list += namelen + 1; + } + } + free(new_name); + } - return 0; -} - -/** - * Remove an extended attribute. - * @param d File/directory to operate on - * @param name Extended attribute name to delete - * @param vol LTFS volume - * @return 0 on success or a negative value on error - */ -int xattr_remove(struct dentry *d, const char *name, struct ltfs_volume *vol) -{ - int ret; - bool is_worm_cart = false; - - CHECK_ARG_NULL(d, -LTFS_NULL_ARG); - CHECK_ARG_NULL(name, -LTFS_NULL_ARG); - CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); - - ret = _xattr_lock_dentry(name, true, d, vol); - if (ret < 0) - return ret; - - ret = tape_get_worm_status(vol->device, &is_worm_cart); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 17237E, "remove xattr: cart stat"); - ret = -LTFS_XATTR_ERR; - goto out_dunlk; - } - - if ((d->is_immutable || d->is_appendonly) - && (is_worm_cart || !_xattr_is_worm_ea(name))) { - /* EA cannot be removed in case of immutable/appendonly */ - ltfsmsg(LTFS_ERR, 17237E, "remove xattr: WORM entry"); - ret = -LTFS_RDONLY_XATTR; - goto out_dunlk; - } - - /* If this xattr is virtual, try the virtual removal function. */ - if (_xattr_is_virtual(d, name, vol)) { - ret = _xattr_remove_virtual(d, name, vol); - if (ret == -LTFS_NO_XATTR) - ret = -LTFS_RDONLY_XATTR; /* non-removable virtual xattr */ - goto out_dunlk; - } - - ret = xattr_do_remove(d, name, false, vol); - if (ret < 0) - goto out_dunlk; - - if (!strcmp(name, "ltfs.vendor.IBM.immutable")) { - d->is_immutable = false; - ltfsmsg(LTFS_INFO, 17238I, "immutable", d->is_immutable, d->name.name); - } - else if (!strcmp(name, "ltfs.vendor.IBM.appendonly")) { - d->is_appendonly = false; - ltfsmsg(LTFS_INFO, 17238I, "appendonly", d->is_appendonly, d->name.name); - } - - d->dirty = true; - ltfs_set_index_dirty(true, false, vol->index); - -out_dunlk: - _xattr_unlock_dentry(name, true, d, vol); - return ret; -} - - -/** - * set LTFS_LIVELINK_EA_NAME - * @param path file path - * @param d File operate on - * @param vol LTFS volume - * @return 0 on success or a negative value on error - */ -int xattr_set_mountpoint_length(struct dentry *d, const char* value, size_t size ) -{ -#ifdef POSIXLINK_ONLY - return 0; -#else - int ret=0; - struct xattr_info *xattr; - - CHECK_ARG_NULL(d, -LTFS_NULL_ARG); - CHECK_ARG_NULL(value, -LTFS_NULL_ARG); - - acquireread_mrsw(&d->meta_lock); - ret = _xattr_seek(&xattr, d, LTFS_LIVELINK_EA_NAME); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 11129E, ret); - releaseread_mrsw(&d->meta_lock); - goto out_set; - } - ret = xattr_do_set(d, LTFS_LIVELINK_EA_NAME, value, size, xattr); - releaseread_mrsw(&d->meta_lock); - -out_set: - return ret; -#endif -} - - -/** - * Search for an xattr with the given name. Must call this function with a lock on - * the dentry's meta_lock. - * @param out On success, points to the xattr that was found. - * @param d Dentry to search. - * @param name Name to search for. - * @return 1 if xattr was found, 0 if not, or a negative value on error. - */ -int _xattr_seek(struct xattr_info **out, struct dentry *d, const char *name) -{ - struct xattr_info *entry; - - *out = NULL; - TAILQ_FOREACH(entry, &d->xattrlist, list) { - if (! strcmp(entry->key.name, name)) { - *out = entry; - break; - } - } - - if (*out) - return 1; - else - return 0; -} - -/** - * Take the volume lock and dentry contents_lock as appropriate for the EA and the access type. - * @param name EA name being read or written. - * @param modify True if EA will be set or deleted, false if it will be read. - * @param d Dentry under consideration. - * @param vol LTFS volume. - * @return 0 on success or -LTFS_REVAL_FAILED if the medium is invalid. - */ -int _xattr_lock_dentry(const char *name, bool modify, struct dentry *d, struct ltfs_volume *vol) -{ - /* EAs that read the extent list need to take the contents_lock */ - if (! strcmp(name, "ltfs.startblock") - || ! strcmp(name, "ltfs.partition")) { - acquireread_mrsw(&d->contents_lock); - } - - /* Other EAs either need no additional locks, or they need the meta_lock. - * The caller is responsible for taking the meta_lock as necessary. */ - return 0; -} - -/** - * Undo locking performed in _xattr_lock_dentry. - * @param name EA name being read or written. - * @param modify True if EA was set or deleted, false if it was read. - * @param d Dentry under consideration. - * @param vol LTFS volume. - */ -void _xattr_unlock_dentry(const char *name, bool modify, struct dentry *d, struct ltfs_volume *vol) -{ - /* EAs that read the extent list need to take the contents_lock */ - if (! strcmp(name, "ltfs.startblock") - || ! strcmp(name, "ltfs.partition")) { - releaseread_mrsw(&d->contents_lock); - } -} - -/** - * Strip a Linux namespace prefix from the given xattr name and return the position of the suffix. - * If the name is "user.X", return the "X" portion. Otherwise, return an error. - * This function does nothing on Mac OS X. - * @param name Name to strip. - * @return A pointer to the name suffix, or NULL to indicate an invalid name. On Mac OS X, - * always returns @name. - */ -const char *_xattr_strip_name(const char *name) -{ -#if (defined (__APPLE__) || defined (mingw_PLATFORM)) - return name; -#else - if (strstr(name, "user.") == name) - return name + 5; - else - return NULL; -#endif -} - -/** - * List real extended attributes for a dentry. Must be called with a read lock held on - * the dentry's meta_lock. - * @param d Dentry to list. - * @param list Output buffer. - * @param size Output buffer size, may be 0. - * @return Number of bytes in listed extended attributes, or a negative value on error. - */ -int _xattr_list_physicals(struct dentry *d, char *list, size_t size) -{ - struct xattr_info *entry; - char *prefix = "\0", *new_name; - int prefixlen = 0, namelen; - int ret = 0, nbytes = 0; - -#if ((!defined (__APPLE__)) && (!defined (mingw_PLATFORM))) - ret = pathname_unformat("user.", &prefix); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 11141E, ret); - return ret; - } - prefixlen = strlen(prefix); -#endif /* (!defined (__APPLE__)) && (!defined (mingw_PLATFORM)) */ - - TAILQ_FOREACH(entry, &d->xattrlist, list) { - ret = pathname_unformat(entry->key.name, &new_name); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 11142E, ret); - goto out; - } - - if(strncmp(new_name, LTFS_LIVELINK_EA_NAME, strlen(LTFS_LIVELINK_EA_NAME) + 1)) { - namelen = strlen(new_name); - - nbytes += prefixlen + namelen + 1; - if (size && (size_t)nbytes <= size) { - memcpy(list, prefix, prefixlen); - list += prefixlen; - memcpy(list, new_name, namelen); - list += namelen + 1; - } - } - free(new_name); - } - -out: -#if ((!defined (__APPLE__)) && (!defined (mingw_PLATFORM))) - free(prefix); -#endif /* (!defined (__APPLE__)) && (!defined (mingw_PLATFORM)) */ - if (ret < 0) - return ret; - return nbytes; +out: +#if ((!defined (__APPLE__)) && (!defined (mingw_PLATFORM))) + free(prefix); +#endif /* (!defined (__APPLE__)) && (!defined (mingw_PLATFORM)) */ + if (ret < 0) + return ret; + return nbytes; } /** @@ -718,7 +404,7 @@ int _xattr_list_physicals(struct dentry *d, char *list, size_t size) * @param vol LTFS volume to which the dentry belongs. * @return true if the name exists and is virtual, false otherwise. */ -bool _xattr_is_virtual(struct dentry *d, const char *name, struct ltfs_volume *vol) +static bool _xattr_is_virtual(struct dentry *d, const char *name, struct ltfs_volume *vol) { /* xattrs on all dentries */ if (! strcmp(name, "ltfs.createTime") @@ -796,6 +482,7 @@ bool _xattr_is_virtual(struct dentry *d, const char *name, struct ltfs_volume *v || ! strcmp(name, "ltfs.mediaIndexPartitionAvailableSpace") || ! strcmp(name, "ltfs.mediaEncrypted") || ! strcmp(name, "ltfs.mediaPool.additionalInfo") + || ! strcmp(name, "ltfs.mediaPool.name") || ! strcmp(name, "ltfs.driveEncryptionState") || ! strcmp(name, "ltfs.driveEncryptionMethod") /* Vendor specific EAs */ @@ -805,6 +492,10 @@ bool _xattr_is_virtual(struct dentry *d, const char *name, struct ltfs_volume *v || ! strcmp(name, "ltfs.vendor.IBM.cartridgeMountNode") || ! strcmp(name, "ltfs.vendor.IBM.logLevel") || ! strcmp(name, "ltfs.vendor.IBM.syslogLevel") + || ! strcmp(name, "ltfs.vendor.IBM.rao") + || ! strcmp(name, "ltfs.vendor.IBM.logPage") + || ! strcmp(name, "ltfs.vendor.IBM.mediaMAM") + || ! strcmp(name, "ltfs.vendor.IBM.dumpincj") || ! strncmp(name, "ltfs.vendor", strlen("ltfs.vendor"))) return true; } @@ -823,8 +514,8 @@ bool _xattr_is_virtual(struct dentry *d, const char *name, struct ltfs_volume *v * -LTFS_NO_XATTR if no such readable virtual xattr exists, * -LTFS_RDONLY_XATTR for write-only virtual EAs, or another negative value on error. */ -int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char *name, - struct ltfs_volume *vol) +static int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char *name, + struct ltfs_volume *vol) { int ret = -LTFS_NO_XATTR; char *val = NULL; @@ -883,24 +574,24 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char } else if (! strcmp(name, "ltfs.driveCaptureDump")) { ret = tape_takedump_drive(vol->device, true); } else if (! strcmp(name, "ltfs.fileUID")) { - ret = _xattr_get_u64(d->uid, &val, name); + ret = xattr_get_u64(d->uid, &val, name); } else if (! strcmp(name, "ltfs.volumeUUID")) { - ret = _xattr_get_string(vol->label->vol_uuid, &val, name); + ret = xattr_get_string(vol->label->vol_uuid, &val, name); } else if (! strcmp(name, "ltfs.volumeName")) { ltfs_mutex_lock(&vol->index->dirty_lock); - ret = _xattr_get_string(vol->index->volume_name.name, &val, name); + ret = xattr_get_string(vol->index->volume_name.name, &val, name); ltfs_mutex_unlock(&vol->index->dirty_lock); } else if (! strcmp(name, "ltfs.softwareVersion")) { - ret = _xattr_get_string(PACKAGE_VERSION, &val, name); + ret = xattr_get_string(PACKAGE_VERSION, &val, name); } else if (! strcmp(name, "ltfs.softwareFormatSpec")) { - ret = _xattr_get_string(LTFS_INDEX_VERSION_STR, &val, name); + ret = xattr_get_string(LTFS_INDEX_VERSION_STR, &val, name); } else if (! strcmp(name, "ltfs.softwareVendor")) { - ret = _xattr_get_string(LTFS_VENDOR_NAME, &val, name); + ret = xattr_get_string(LTFS_VENDOR_NAME, &val, name); } else if (! strcmp(name, "ltfs.softwareProduct")) { if ( strncmp( PACKAGE_VERSION, "1", 1 )==0 ) - ret = _xattr_get_string("LTFS SDE", &val, name); + ret = xattr_get_string("LTFS SDE", &val, name); else if ( strncmp( PACKAGE_VERSION, "2", 1 )==0 ) - ret = _xattr_get_string("LTFS LE", &val, name); + ret = xattr_get_string("LTFS LE", &val, name); else ret = -LTFS_NO_XATTR; } else if (! strcmp(name, "ltfs.vendor.IBM.logLevel")) { @@ -997,7 +688,7 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char val[1] = '\0'; } } else if (! strcmp(name, "ltfs.startblock")) { - ret = _xattr_get_u64(TAILQ_FIRST(&d->extentlist)->start.block, &val, name); + ret = xattr_get_u64(TAILQ_FIRST(&d->extentlist)->start.block, &val, name); } } @@ -1005,10 +696,10 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char if (ret == -LTFS_NO_XATTR && d == vol->index->root) { if (! strcmp(name, "ltfs.commitMessage")) { ltfs_mutex_lock(&vol->index->dirty_lock); - ret = _xattr_get_string(vol->index->commit_message, &val, name); + ret = xattr_get_string(vol->index->commit_message, &val, name); ltfs_mutex_unlock(&vol->index->dirty_lock); } else if (! strcmp(name, "ltfs.volumeSerial")) { - ret = _xattr_get_string(vol->label->barcode, &val, name); + ret = xattr_get_string(vol->label->barcode, &val, name); } else if (! strcmp(name, "ltfs.volumeFormatTime")) { ret = _xattr_get_time(&vol->label->format_time, &val, name); if (ret == LTFS_TIME_OUT_OF_RANGE) { @@ -1016,9 +707,9 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char ret = 0; } } else if (! strcmp(name, "ltfs.volumeBlocksize")) { - ret = _xattr_get_u64(vol->label->blocksize, &val, name); + ret = xattr_get_u64(vol->label->blocksize, &val, name); } else if (! strcmp(name, "ltfs.indexGeneration")) { - ret = _xattr_get_u64(vol->index->generation, &val, name); + ret = xattr_get_u64(vol->index->generation, &val, name); } else if (! strcmp(name, "ltfs.indexTime")) { ret = _xattr_get_time(&vol->index->mod_time, &val, name); if (ret == LTFS_TIME_OUT_OF_RANGE) { @@ -1026,22 +717,22 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char ret = 0; } } else if (! strcmp(name, "ltfs.policyExists")) { - ret = _xattr_get_string(ic->have_criteria ? "true" : "false", &val, name); + ret = xattr_get_string(ic->have_criteria ? "true" : "false", &val, name); } else if (! strcmp(name, "ltfs.policyAllowUpdate")) { - ret = _xattr_get_string(vol->index->criteria_allow_update ? "true" : "false", + ret = xattr_get_string(vol->index->criteria_allow_update ? "true" : "false", &val, name); } else if (! strcmp(name, "ltfs.policyMaxFileSize") && ic->have_criteria) { - ret = _xattr_get_u64(ic->max_filesize_criteria, &val, name); + ret = xattr_get_u64(ic->max_filesize_criteria, &val, name); } else if (! strcmp(name, "ltfs.volumeCompression")) { - ret = _xattr_get_string(vol->label->enable_compression ? "true" : "false", &val, name); + ret = xattr_get_string(vol->label->enable_compression ? "true" : "false", &val, name); } else if (! strcmp(name, "ltfs.indexLocation")) { ret = _xattr_get_tapepos(&vol->index->selfptr, &val, name); } else if (! strcmp(name, "ltfs.indexPrevious")) { ret = _xattr_get_tapepos(&vol->index->backptr, &val, name); } else if (! strcmp(name, "ltfs.indexCreator")) { - ret = _xattr_get_string(vol->index->creator, &val, name); + ret = xattr_get_string(vol->index->creator, &val, name); } else if (! strcmp(name, "ltfs.labelCreator")) { - ret = _xattr_get_string(vol->label->creator, &val, name); + ret = xattr_get_string(vol->label->creator, &val, name); } else if (! strcmp(name, "ltfs.indexVersion")) { ltfs_mutex_lock(&vol->index->dirty_lock); ret = _xattr_get_version(vol->index->version, &val, name); @@ -1099,18 +790,27 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char } else if (! strcmp(name, "ltfs.mediaIndexPartitionAvailableSpace")) { ret = _xattr_get_cartridge_capacity(&cap, &cap.remaining_ip, &val, name, vol); } else if (! strcmp(name, "ltfs.mediaEncrypted")) { - ret = _xattr_get_string(tape_get_media_encrypted(vol->device), &val, name); + ret = xattr_get_string(tape_get_media_encrypted(vol->device), &val, name); + } else if (! strcmp(name, "ltfs.mediaPool.name")) { + char *tmp=NULL; + ret = tape_get_media_pool_info(vol, &val, &tmp); + if (tmp) + free(tmp); + if (ret < 0 || !val) + ret = -LTFS_NO_XATTR; } else if (! strcmp(name, "ltfs.mediaPool.additionalInfo")) { char *tmp=NULL; ret = tape_get_media_pool_info(vol, &tmp, &val); + if (tmp) + free(tmp); if (ret < 0 || !val) ret = -LTFS_NO_XATTR; } else if (! strcmp(name, "ltfs.driveEncryptionState")) { - ret = _xattr_get_string(tape_get_drive_encryption_state(vol->device), &val, name); + ret = xattr_get_string(tape_get_drive_encryption_state(vol->device), &val, name); } else if (! strcmp(name, "ltfs.driveEncryptionMethod")) { - ret = _xattr_get_string(tape_get_drive_encryption_method(vol->device), &val, name); + ret = xattr_get_string(tape_get_drive_encryption_method(vol->device), &val, name); } else if (! strcmp(name, "ltfs.vendor.IBM.referencedBlocks")) { - ret = _xattr_get_u64(ltfs_get_valid_block_count_unlocked(vol), &val, name); + ret = xattr_get_u64(ltfs_get_valid_block_count_unlocked(vol), &val, name); } else if (! strcmp(name, "ltfs.vendor.IBM.trace")) { ret = ltfs_get_trace_status(&val); } else if (! strcmp(name, "ltfs.vendor.IBM.totalBlocks")) { @@ -1118,7 +818,7 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char if (ret < 0) val = NULL; else - ret = _xattr_get_u64(append_pos, &val, name); + ret = xattr_get_u64(append_pos, &val, name); } else if (! strcmp(name, "ltfs.vendor.IBM.cartridgeMountNode")) { ret = asprintf(&val, "localhost"); if (ret < 0) { @@ -1126,12 +826,63 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char val = NULL; ret = -LTFS_NO_MEMORY; } + } else if ( (!strncmp(name, "ltfs.vendor.IBM.logPage.", strlen("ltfs.vendor.IBM.logPage."))) && + (strlen(name) == strlen("ltfs.vendor.IBM.logPage.XX.XX")) ) { + char page_str[3] = {0x00, 0x00, 0x00}; + char subpage_str[3] = {0x00, 0x00, 0x00}; + + uint8_t page = 0xFF; + uint8_t subpage = 0xFF; + + char *endptr = NULL; + + page_str[0] = name[24]; + page_str[1] = name[25]; + subpage_str[0] = name[27]; + subpage_str[1] = name[28]; + + page = (uint8_t)(strtoul(page_str, &endptr, 16)); + if (*endptr) return -LTFS_NO_XATTR; + + subpage = (uint8_t)(strtoul(subpage_str, &endptr, 16)); + if (*endptr) return -LTFS_NO_XATTR; + + ret = ltfs_logpage(page, subpage, (unsigned char *)buf, buf_size, vol); + + } else if ( (!strncmp(name, "ltfs.vendor.IBM.mediaMAM.", strlen("ltfs.vendor.IBM.mediaMAM."))) && + (strlen(name) == strlen("ltfs.vendor.IBM.mediaMAM.XX")) ) { + char part_str[3] = {0x00, 0x00, 0x00}; + tape_partition_t part = 0; + + char *endptr = NULL; + + part_str[0] = name[25]; + part_str[1] = name[26]; + + if (!strncmp(part_str, "IP", sizeof(part_str))) { + part = ltfs_part_id2num(vol->label->partid_ip, vol); + } else if (!strncmp(part_str, "DP", sizeof(part_str))) { + part = ltfs_part_id2num(vol->label->partid_dp, vol);; + } else { + part = (uint8_t)(strtoul(part_str, &endptr, 16)); + if (*endptr) return -LTFS_NO_XATTR; + } + + if (part > 1) return -LTFS_NO_XATTR; + + ret = ltfs_mam(part, (unsigned char *)buf, buf_size, vol); + + } else if (! strcmp(name, "ltfs.vendor.IBM.dumpincj")) { + incj_dump(vol); + ret = 0; + } else if (! strncmp(name, "ltfs.vendor", strlen("ltfs.vendor"))) { if (! strncmp(name + strlen("ltfs.vendor."), LTFS_VENDOR_NAME, strlen(LTFS_VENDOR_NAME))) { ret = _xattr_get_vendorunique_xattr(&val, name, vol); } } else if (! strcmp(name, "ltfs.sync")) { - ret = ltfs_sync_index(SYNC_EA, false, vol); + ltfs_set_commit_message_reason(SYNC_EA, vol); + ret = ltfs_sync_index(SYNC_EA, false, LTFS_FULL_INDEX, vol); } } @@ -1160,14 +911,17 @@ int _xattr_get_virtual(struct dentry *d, char *buf, size_t buf_size, const char * @return 0 on success, -LTFS_NO_XATTR if the xattr is not a settable virtual xattr, * or another negative value on error. */ -int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, - size_t size, struct ltfs_volume *vol) +static int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, + size_t size, struct ltfs_volume *vol) { int ret = 0; + enum ltfs_index_type idx_type = LTFS_INDEX_AUTO; - if (! strcmp(name, "ltfs.sync") && d == vol->index->root) - ret = ltfs_sync_index(SYNC_EA, false, vol); - else if (! strcmp(name, "ltfs.commitMessage") && d == vol->index->root) { + if ((! strcmp(name, "ltfs.commitMessage") || + ! strcmp(name, "ltfs.sync") || + ! strcmp(name, "ltfs.vendor.IBM.FullSync") || + ! strcmp(name, "ltfs.vendor.IBM.IncrementalSync")) + && d == vol->index->root) { char *value_null_terminated, *new_value; if (size > INDEX_MAX_COMMENT_LEN) { @@ -1175,39 +929,61 @@ int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, ret = -LTFS_LARGE_XATTR; } +#ifdef FORMAT_SPEC25 + if (! strcmp(name, "ltfs.vendor.IBM.FullSync")) + idx_type = LTFS_FULL_INDEX; + else if (! strcmp(name, "ltfs.vendor.IBM.IncrementalSync")) + idx_type = LTFS_INCREMENTAL_INDEX; +#else + if (! strcmp(name, "ltfs.vendor.IBM.FullSync") || + ! strcmp(name, "ltfs.vendor.IBM.IncrementalSync")) { + } +#endif + ltfs_mutex_lock(&vol->index->dirty_lock); - if (! value || ! size) { - /* Clear the current comment field */ - if (vol->index->commit_message) { - free(vol->index->commit_message); - vol->index->commit_message = NULL; - } + if (! vol->index->dirty) { + /* Do nothing because index is clean */ + ret = 0; } else { - value_null_terminated = malloc(size + 1); - if (! value_null_terminated) { - ltfsmsg(LTFS_ERR, 10001E, "_xattr_set_virtual: commit_message"); - ltfs_mutex_unlock(&vol->index->dirty_lock); - return -LTFS_NO_MEMORY; - } - memcpy(value_null_terminated, value, size); - value_null_terminated[size] = '\0'; + if (! value || ! size) { + /* Clear the current comment field */ + if (vol->index->commit_message) { + free(vol->index->commit_message); + vol->index->commit_message = NULL; + } + } else { + value_null_terminated = malloc(size + 1); + if (! value_null_terminated) { + ltfsmsg(LTFS_ERR, 10001E, "_xattr_set_virtual: commit_message"); + ltfs_mutex_unlock(&vol->index->dirty_lock); + return -LTFS_NO_MEMORY; + } + memcpy(value_null_terminated, value, size); + value_null_terminated[size] = '\0'; - ret = pathname_format(value_null_terminated, &new_value, false, true); - free(value_null_terminated); - if (ret < 0) { - ltfs_mutex_unlock(&vol->index->dirty_lock); - return ret; + ret = pathname_format(value_null_terminated, &new_value, false, true); + free(value_null_terminated); + if (ret < 0) { + /* Try to sync index even if the value is not valid */ + ltfs_set_commit_message_reason_unlocked(SYNC_EA, vol); + ltfs_mutex_unlock(&vol->index->dirty_lock); + + ret = ltfs_sync_index(SYNC_EA, false, LTFS_FULL_INDEX, vol); + return ret; + } + ret = 0; + + /* Update the commit message in the index */ + if (vol->index->commit_message) + free(vol->index->commit_message); + vol->index->commit_message = new_value; } - ret = 0; - /* Update the commit message in the index */ - if (vol->index->commit_message) - free(vol->index->commit_message); - vol->index->commit_message = new_value; + ltfs_set_index_dirty(false, false, vol->index); } - ltfs_set_index_dirty(false, false, vol->index); ltfs_mutex_unlock(&vol->index->dirty_lock); + ret = ltfs_sync_index(SYNC_EA, false, idx_type, vol); } else if (! strcmp(name, "ltfs.volumeName") && d == vol->index->root) { char *value_null_terminated, *new_value; @@ -1339,6 +1115,16 @@ int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, } else ret = -LTFS_STRING_CONVERSION; free(v); + } else if (! strcmp(name, "ltfs.vendor.IBM.rao")) { + char *v; + v = strndup(value, size); + if (! v) { + ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); + return -LTFS_NO_MEMORY; + } + if (strlen(v) > PATH_MAX) return -LTFS_LARGE_XATTR; /* file path size check */ + ret = ltfs_get_rao_list(v, vol); + free(v); } else if (! strcmp(name, "ltfs.vendor.IBM.trace")) { char *v; @@ -1416,7 +1202,7 @@ int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, lock = strtoull(v, &invalid_start, 0); if( (*invalid_start == '\0') && v ) { - mam_lockval new = UNLOCKED_MAM; + mam_lockval_t new = UNLOCKED_MAM; char status_mam[TC_MAM_LOCKED_MAM_SIZE]; switch (vol->t_attr->vollock) { @@ -1468,13 +1254,15 @@ int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, vol->lock_status = new; ltfs_set_index_dirty(false, false, vol->index); - ret = ltfs_sync_index(SYNC_ADV_LOCK, false, vol); + ltfs_set_commit_message_reason_unlocked(SYNC_ADV_LOCK, vol); + + ret = ltfs_sync_index(SYNC_ADV_LOCK, false, LTFS_FULL_INDEX, vol); ret = tape_device_lock(vol->device); if (ret < 0) { ltfsmsg(LTFS_ERR, 12010E, __FUNCTION__); return ret; } - ret = ltfs_write_index(ltfs_ip_id(vol), SYNC_EA, vol); + ret = ltfs_write_index(ltfs_ip_id(vol), SYNC_EA, LTFS_FULL_INDEX, vol); tape_device_unlock(vol->device); } else ret = -LTFS_STRING_CONVERSION; @@ -1482,6 +1270,8 @@ int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, free(v); } else if (! strcmp(name, "ltfs.mediaPool.additionalInfo")) { ret = tape_set_media_pool_info(vol, value, size, false); + } else if (! strcmp(name, "ltfs.mediaPool.name")) { + ret = tape_set_media_pool_info(vol, value, size, true); } else ret = -LTFS_NO_XATTR; @@ -1497,7 +1287,7 @@ int _xattr_set_virtual(struct dentry *d, const char *name, const char *value, * @return 0 on success, -LTFS_NO_XATTR if the xattr is not a removable virtual xattr, or another * negative value on error. */ -int _xattr_remove_virtual(struct dentry *d, const char *name, struct ltfs_volume *vol) +static int _xattr_remove_virtual(struct dentry *d, const char *name, struct ltfs_volume *vol) { int ret = 0; @@ -1527,201 +1317,519 @@ int _xattr_remove_virtual(struct dentry *d, const char *name, struct ltfs_volume return ret; } -int _xattr_get_cartridge_health(cartridge_health_info *h, int64_t *val, char **outval, - const char *msg, struct ltfs_volume *vol) +/* Global functions */ + +int xattr_get_string(const char *val, char **outval, const char *msg) { - int ret = ltfs_get_cartridge_health(h, vol); - if (ret == 0) { - ret = asprintf(outval, "%"PRId64, *val); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - *outval = NULL; - return -LTFS_NO_MEMORY; - } - } else + if (! val) + return 0; + *outval = strdup(val); + if (! (*outval)) { + ltfsmsg(LTFS_ERR, 10001E, msg); + return -LTFS_NO_MEMORY; + } + return 0; +} + +int xattr_get_u64(uint64_t val, char **outval, const char *msg) +{ + int ret = asprintf(outval, "%"PRIu64, val); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, msg); *outval = NULL; + ret = -LTFS_NO_MEMORY; + } return ret; } -int _xattr_get_cartridge_health_u64(cartridge_health_info *h, uint64_t *val, char **outval, - const char *msg, struct ltfs_volume *vol) +int xattr_do_set(struct dentry *d, const char *name, const char *value, size_t size, + struct xattr_info *xattr) { - int ret = ltfs_get_cartridge_health(h, vol); - if (ret == 0 && (int64_t)(*val) != UNSUPPORTED_CARTRIDGE_HEALTH) { - ret = asprintf(outval, "%"PRIu64, *val); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - *outval = NULL; + int ret = 0; + + /* clear existing xattr or set up new one */ + if (xattr) { + if (xattr->value) { + free(xattr->value); + xattr->value = NULL; + } + } else { + xattr = (struct xattr_info *) calloc(1, sizeof(struct xattr_info)); + if (! xattr) { + ltfsmsg(LTFS_ERR, 10001E, "xattr_do_set: xattr"); + return -LTFS_NO_MEMORY; + } + xattr->key.name = strdup(name); + if (! xattr->key.name) { + ltfsmsg(LTFS_ERR, 10001E, "xattr_do_set: xattr key"); ret = -LTFS_NO_MEMORY; + goto out_free; } - } else if (ret == 0) { - ret = asprintf(outval, "%"PRId64, UNSUPPORTED_CARTRIDGE_HEALTH); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - *outval = NULL; + xattr->key.percent_encode = fs_is_percent_encode_required(xattr->key.name); + TAILQ_INSERT_HEAD(&d->xattrlist, xattr, list); + } + + /* copy new value */ + xattr->size = size; + if (size > 0) { + xattr->value = (char *)malloc(size); + if (! xattr->value) { + ltfsmsg(LTFS_ERR, 10001E, "xattr_do_set: xattr value"); ret = -LTFS_NO_MEMORY; + goto out_remove; } - } else - *outval = NULL; + memcpy(xattr->value, value, size); + } + return 0; + +out_remove: + TAILQ_REMOVE(&d->xattrlist, xattr, list); +out_free: + if (xattr->key.name) + free(xattr->key.name); + free(xattr); return ret; } -int _xattr_get_cartridge_capacity(struct device_capacity *cap, unsigned long *val, char **outval, - const char *msg, struct ltfs_volume *vol) +/** + * Set an extended attribute. + * @param d File or directory to set the xattr on. + * @param name Name to set. + * @param value Value to set, may be binary, not necessarily null-terminated. + * @param size Size of value in bytes. + * @param flags XATTR_REPLACE to fail if xattr doesn't exist, XATTR_CREATE to fail if it does + * exist, or 0 to ignore any existing value. + * @return 0 on success or a negative value on error. + */ +int xattr_set(struct dentry *d, const char *name, const char *value, size_t size, + int flags, struct ltfs_volume *vol) { - double scale = vol->label->blocksize / 1048576.0; - int ret = ltfs_capacity_data_unlocked(cap, vol); - if (ret == 0) { - ret = asprintf(outval, "%lu", (unsigned long)((*val) * scale)); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - *outval = NULL; - return -LTFS_NO_MEMORY; + struct xattr_info *xattr; + bool replace, create; + int ret; + bool is_worm_cart = false; + bool disable_worm_ea = false; + char *new_value="1"; + bool write_idx = false; + + CHECK_ARG_NULL(d, -LTFS_NULL_ARG); + CHECK_ARG_NULL(name, -LTFS_NULL_ARG); + CHECK_ARG_NULL(value, -LTFS_NULL_ARG); + CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); + if (size > LTFS_MAX_XATTR_SIZE) + return -LTFS_LARGE_XATTR; /* this is the error returned by ext3 when the xattr is too large */ + + replace = flags & XATTR_REPLACE; + create = flags & XATTR_CREATE; + + ret = _xattr_lock_dentry(name, true, d, vol); + if (ret < 0) + return ret; + + ret = tape_get_worm_status(vol->device, &is_worm_cart); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17237E, "set xattr: cart stat"); + ret = -LTFS_XATTR_ERR; + goto out_unlock; + } + + if ((is_worm_cart && (d->is_immutable || (d->is_appendonly && strcmp(name, "ltfs.vendor.IBM.immutable")))) + || (!is_worm_cart && (d->is_immutable || d->is_appendonly) && !_xattr_is_worm_ea(name))) { + /* EA cannot be set in case of immutable/appendonly */ + ltfsmsg(LTFS_ERR, 17237E, "set xattr: WORM entry"); + ret = -LTFS_RDONLY_XATTR; + goto out_unlock; + } + + /* Check if this is a user-writeable virtual xattr */ + if (_xattr_is_virtual(d, name, vol)) { + ret = _xattr_set_virtual(d, name, value, size, vol); + if (ret == -LTFS_NO_XATTR) + ret = -LTFS_RDONLY_XATTR; + goto out_unlock; + } + + /* In the future, there could be user-writeable reserved xattrs. For now, just deny + * writes to all reserved xattrs not covered by the user-writeable virtual xattrs above. */ + if (strcasestr(name, "ltfs") == name && !_xattr_is_stored_vea(name) && !_xattr_is_worm_ea(name)) { + ret = -LTFS_RDONLY_XATTR; + goto out_unlock; + } + + acquirewrite_mrsw(&d->meta_lock); + + /* Search for existing xattr with this name. */ + ret = _xattr_seek(&xattr, d, name); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 11122E, ret); + releasewrite_mrsw(&d->meta_lock); + goto out_unlock; + } + if (create && xattr) { + releasewrite_mrsw(&d->meta_lock); + ret = -LTFS_XATTR_EXISTS; + goto out_unlock; + } else if (replace && ! xattr) { + releasewrite_mrsw(&d->meta_lock); + ret = -LTFS_NO_XATTR; + goto out_unlock; + } + if (_xattr_is_worm_ea(name)) { + disable_worm_ea = (strncmp(value, "0", size) == 0); + + if (is_worm_cart && disable_worm_ea) { + ltfsmsg(LTFS_ERR, 17237E, "set xattr: clear WORM"); + releasewrite_mrsw(&d->meta_lock); + ret = -LTFS_XATTR_ERR; + goto out_unlock; + } + if (!disable_worm_ea) { + /* All values other than 0 is treated as 1 */ + value = new_value; + size = strlen(new_value); } + } + + /* Set extended attribute */ + ret = xattr_do_set(d, name, value, size, xattr); + if (ret < 0) { + releasewrite_mrsw(&d->meta_lock); + goto out_unlock; + } + + /* update metadata */ + if (!strcmp(name, "ltfs.vendor.IBM.immutable")) { + d->is_immutable = !disable_worm_ea; + ltfsmsg(LTFS_INFO, 17238I, "immutable", d->is_immutable, d->name.name); + } + else if (!strcmp(name, "ltfs.vendor.IBM.appendonly")) { + d->is_appendonly = !disable_worm_ea; + ltfsmsg(LTFS_INFO, 17238I, "appendonly", d->is_appendonly, d->name.name); + } + + get_current_timespec(&d->change_time); + + ltfs_set_index_dirty(true, false, vol->index); + ltfs_set_dentry_dirty(d, vol); + + releasewrite_mrsw(&d->meta_lock); + + if (write_idx) { + ltfs_set_commit_message_reason(SYNC_EA, vol); + ret = ltfs_sync_index(SYNC_EA, false, LTFS_INDEX_AUTO, vol); } else - *outval = NULL; + ret = 0; + +out_unlock: + _xattr_unlock_dentry(name, true, d, vol); return ret; } -int _xattr_get_time(struct ltfs_timespec *val, char **outval, const char *msg) +/** + * Get an extended attribute. Returns an error if the provided buffer is not large enough + * to contain the attribute value. + * @param d File/directory to check + * @param name Xattr name + * @param value On success, contains xattr value + * @param size Output buffer size in bytes + * @param vol LTFS volume + * @return if size is nonzero, number of bytes returned in the value buffer. if size is zero, + * number of bytes in the xattr value. returns a negative value on error. If the + * operation needs to be restarted, then -LTFS_RESTART_OPERATION is returned instead. + */ +int xattr_get(struct dentry *d, const char *name, char *value, size_t size, + struct ltfs_volume *vol) { + struct xattr_info *xattr = NULL; int ret; - ret = xml_format_time(*val, outval); - if (! (*outval)) { - ltfsmsg(LTFS_ERR, 11145E, msg); - return -LTFS_NO_MEMORY; + CHECK_ARG_NULL(d, -LTFS_NULL_ARG); + CHECK_ARG_NULL(name, -LTFS_NULL_ARG); + CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); + if (size > 0 && ! value) { + ltfsmsg(LTFS_ERR, 11123E); + return -LTFS_BAD_ARG; } - return ret; -} + ret = _xattr_lock_dentry(name, false, d, vol); + if (ret < 0) + return ret; + + /* Try to get a virtual xattr first. */ + if (_xattr_is_virtual(d, name, vol)) { + + if (vol->mount_type == MOUNT_ROLLBACK_META) { + _xattr_unlock_dentry(name, false, d, vol); + return -LTFS_DEVICE_UNREADY; + } + + ret = _xattr_get_virtual(d, value, size, name, vol); + if (ret == -LTFS_DEVICE_FENCED) { + _xattr_unlock_dentry(name, false, d, vol); + ret = ltfs_wait_revalidation(vol); + return (ret == 0) ? -LTFS_RESTART_OPERATION : ret; + } else if (NEED_REVAL(ret)) { + _xattr_unlock_dentry(name, false, d, vol); + ret = ltfs_revalidate(false, vol); + return (ret == 0) ? -LTFS_RESTART_OPERATION : ret; + } else if (IS_UNEXPECTED_MOVE(ret)) { + vol->reval = -LTFS_REVAL_FAILED; + _xattr_unlock_dentry(name, false, d, vol); + return ret; + }else if (ret != -LTFS_NO_XATTR) { + /* if ltfs.sync is specified, don't print any message */ + if (ret < 0 && ret != -LTFS_RDONLY_XATTR) + ltfsmsg(LTFS_ERR, 11128E, ret); + goto out_unlock; + } + } + + acquireread_mrsw(&d->meta_lock); + + /* Look for a real xattr. */ + ret = _xattr_seek(&xattr, d, name); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 11129E, ret); + releaseread_mrsw(&d->meta_lock); + goto out_unlock; + } + + /* Generate output. */ + ret = 0; + if (! xattr) { + /* There's no such extended attribute */ + ret = -LTFS_NO_XATTR; + } else if (size && xattr->size > size) { + /* There is no space to fill the buffer */ + ret = -LTFS_SMALL_BUFFER; + } else if (size) { + /* Copy the extended attribute to the requester */ + memcpy(value, xattr->value, xattr->size); + ret = xattr->size; + } else /* size is zero */ { + /* Return how many bytes will be necessary to read this xattr */ + ret = xattr->size; + } -int _xattr_get_dentry_time(struct dentry *d, struct ltfs_timespec *val, char **outval, - const char *msg) -{ - int ret; - acquireread_mrsw(&d->meta_lock); - ret = _xattr_get_time(val, outval, msg); releaseread_mrsw(&d->meta_lock); + +out_unlock: + _xattr_unlock_dentry(name, false, d, vol); return ret; } -int _xattr_get_string(const char *val, char **outval, const char *msg) +/** + * Copy a list of extended attribute names to a user-provided buffer. + * @param d File/directory to get the list of extended attributes from + * @param list Output buffer for xattr names + * @param size Output buffer size in bytes + * @param vol LTFS volume + * @return number of bytes in buffer on success, or a negative value on error. + */ +int xattr_list(struct dentry *d, char *list, size_t size, struct ltfs_volume *vol) { - if (! val) - return 0; - *outval = strdup(val); - if (! (*outval)) { - ltfsmsg(LTFS_ERR, 10001E, msg); - return -LTFS_NO_MEMORY; - } - return 0; -} + int ret, nbytes = 0; -int _xattr_get_u64(uint64_t val, char **outval, const char *msg) -{ - int ret = asprintf(outval, "%"PRIu64, val); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - *outval = NULL; - ret = -LTFS_NO_MEMORY; + CHECK_ARG_NULL(d, -LTFS_NULL_ARG); + CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); + if (size > 0 && ! list) { + ltfsmsg(LTFS_ERR, 11130E); + return -LTFS_BAD_ARG; } - return ret; -} -int _xattr_get_tapepos(struct tape_offset *val, char **outval, const char *msg) -{ - int ret = asprintf(outval, "%c:%"PRIu64, val->partition, val->block); + acquireread_mrsw(&d->meta_lock); + + /* Fill the buffer with only real xattrs. */ + if (size) + memset(list, 0, size); + + ret = _xattr_list_physicals(d, list, size); if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - return -LTFS_NO_MEMORY; + ltfsmsg(LTFS_ERR, 11133E, ret); + goto out; } - return 0; + nbytes += ret; + + /* + * There used to be an _xattr_list_virtuals function which was called here. + * Listing virtual xattrs causes problems with files copied from LTFS to another filesystem + * which are attempted to be brought back. Since the copy utility may also copy the + * reserved virtual extended attributes, the copy operation will fail with permission + * denied problems. + */ + + /* Was the buffer large enough? */ + if (size && (size_t)nbytes > size) + ret = -LTFS_SMALL_BUFFER; + +out: + releaseread_mrsw(&d->meta_lock); + if (ret < 0) + return ret; + return nbytes; } -int _xattr_get_partmap(struct ltfs_label *label, char **outval, const char *msg) +/** + * Actually remove an extended attribute. + * @param dentry dentry to operate on + * @param name xattr name to delete + * @param force true to force removal, false to verify namespaces first + * @param vol LTFS volume + * @return 0 on success or a negative value on error + */ +int xattr_do_remove(struct dentry *d, const char *name, bool force, struct ltfs_volume *vol) { - int ret = asprintf(outval, "I:%c,D:%c", label->partid_ip, label->partid_dp); + int ret; + struct xattr_info *xattr; + + acquirewrite_mrsw(&d->meta_lock); + + /* Look for a real extended attribute. */ + ret = _xattr_seek(&xattr, d, name); if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - return -LTFS_NO_MEMORY; + ltfsmsg(LTFS_ERR, 11140E, ret); + releasewrite_mrsw(&d->meta_lock); + return ret; + } else if (! xattr) { + releasewrite_mrsw(&d->meta_lock); + return -LTFS_NO_XATTR; } - return 0; -} -int _xattr_get_version(int version, char **outval, const char *msg) -{ - int ret; - if (version == 10000) { - *outval = strdup("1.0"); - if (! (*outval)) { - ltfsmsg(LTFS_ERR, 10001E, msg); - return -LTFS_NO_MEMORY; - } - } else { - ret = asprintf(outval, "%d.%d.%d", version/10000, (version % 10000)/100, version % 100); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 10001E, msg); - return -LTFS_NO_MEMORY; + if (! force) { + /* If this xattr is in the reserved namespace, the user can't remove it. */ + /* TODO: in the future, there could be user-removable reserved xattrs. */ + if (strcasestr(name, "ltfs") == name && !_xattr_is_stored_vea(name) && !_xattr_is_worm_ea(name)) { + releasewrite_mrsw(&d->meta_lock); + return -LTFS_RDONLY_XATTR; } } + + /* Remove the xattr. */ + TAILQ_REMOVE(&d->xattrlist, xattr, list); + get_current_timespec(&d->change_time); + releasewrite_mrsw(&d->meta_lock); + + free(xattr->key.name); + if (xattr->value) + free(xattr->value); + free(xattr); + return 0; } -int _xattr_set_time(struct dentry *d, struct ltfs_timespec *out, const char *value, size_t size, - const char *msg, struct ltfs_volume *vol) +/** + * Remove an extended attribute. + * @param d File/directory to operate on + * @param name Extended attribute name to delete + * @param vol LTFS volume + * @return 0 on success or a negative value on error + */ +int xattr_remove(struct dentry *d, const char *name, struct ltfs_volume *vol) { int ret; - struct ltfs_timespec t; - char *value_null_terminated; + bool is_worm_cart = false; - value_null_terminated = malloc(size + 1); - if (! value_null_terminated) { - ltfsmsg(LTFS_ERR, 10001E, msg); - return -LTFS_NO_MEMORY; + CHECK_ARG_NULL(d, -LTFS_NULL_ARG); + CHECK_ARG_NULL(name, -LTFS_NULL_ARG); + CHECK_ARG_NULL(vol, -LTFS_NULL_ARG); + + ret = _xattr_lock_dentry(name, true, d, vol); + if (ret < 0) + return ret; + + ret = tape_get_worm_status(vol->device, &is_worm_cart); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17237E, "remove xattr: cart stat"); + ret = -LTFS_XATTR_ERR; + goto out_dunlk; } - memcpy(value_null_terminated, value, size); - value_null_terminated[size] = '\0'; - ret = xml_parse_time(false, value_null_terminated, &t); - free(value_null_terminated); + if ((d->is_immutable || d->is_appendonly) + && (is_worm_cart || !_xattr_is_worm_ea(name))) { + /* EA cannot be removed in case of immutable/appendonly */ + ltfsmsg(LTFS_ERR, 17237E, "remove xattr: WORM entry"); + ret = -LTFS_RDONLY_XATTR; + goto out_dunlk; + } + + /* If this xattr is virtual, try the virtual removal function. */ + if (_xattr_is_virtual(d, name, vol)) { + ret = _xattr_remove_virtual(d, name, vol); + if (ret == -LTFS_NO_XATTR) + ret = -LTFS_RDONLY_XATTR; /* non-removable virtual xattr */ + goto out_dunlk; + } + + ret = xattr_do_remove(d, name, false, vol); if (ret < 0) - return -LTFS_BAD_ARG; + goto out_dunlk; - acquirewrite_mrsw(&d->meta_lock); - *out = t; - d->dirty = true; - releasewrite_mrsw(&d->meta_lock); + if (!strcmp(name, "ltfs.vendor.IBM.immutable")) { + d->is_immutable = false; + ltfsmsg(LTFS_INFO, 17238I, "immutable", d->is_immutable, d->name.name); + } + else if (!strcmp(name, "ltfs.vendor.IBM.appendonly")) { + d->is_appendonly = false; + ltfsmsg(LTFS_INFO, 17238I, "appendonly", d->is_appendonly, d->name.name); + } ltfs_set_index_dirty(true, false, vol->index); + ltfs_set_dentry_dirty(d, vol); + +out_dunlk: + _xattr_unlock_dentry(name, true, d, vol); return ret; } -int _xattr_get_vendorunique_xattr(char **outval, const char *msg, struct ltfs_volume *vol) +/** + * Strip a Linux namespace prefix from the given xattr name and return the position of the suffix. + * If the name is "user.X", return the "X" portion. Otherwise, return an error. + * This function does nothing on Mac OS X. + * @param name Name to strip. + * @return A pointer to the name suffix, or NULL to indicate an invalid name. On Mac OS X, + * always returns @name. + */ +const char *xattr_strip_name(const char *name) { - int ret; - - ret = ltfs_get_vendorunique_xattr(msg, outval, vol); - if (ret != 0) - *outval = NULL; - - return ret; +#if (defined (__APPLE__) || defined (mingw_PLATFORM)) + return name; +#else + if (strstr(name, "user.") == name) + return name + 5; + else + return NULL; +#endif } -int _xattr_set_vendorunique_xattr(const char *name, const char *value, size_t size, struct ltfs_volume *vol) +/** + * set LTFS_LIVELINK_EA_NAME + * @param path file path + * @param d File operate on + * @param vol LTFS volume + * @return 0 on success or a negative value on error + */ +int xattr_set_mountpoint_length(struct dentry *d, const char* value, size_t size ) { - int ret; - - ret = ltfs_set_vendorunique_xattr(name, value, size, vol); +#ifdef POSIXLINK_ONLY + return 0; +#else + int ret=0; + struct xattr_info *xattr; - return ret; -} + CHECK_ARG_NULL(d, -LTFS_NULL_ARG); + CHECK_ARG_NULL(value, -LTFS_NULL_ARG); -bool _xattr_is_worm_ea(const char *name) -{ - if (!strcmp(name, "ltfs.vendor.IBM.immutable") || !strcmp(name, "ltfs.vendor.IBM.appendonly")) { - /* WORM related xattr */ - return true; + acquireread_mrsw(&d->meta_lock); + ret = _xattr_seek(&xattr, d, LTFS_LIVELINK_EA_NAME); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 11129E, ret); + releaseread_mrsw(&d->meta_lock); + goto out_set; } - return false; + ret = xattr_do_set(d, LTFS_LIVELINK_EA_NAME, value, size, xattr); + releaseread_mrsw(&d->meta_lock); + +out_set: + return ret; +#endif } diff --git a/src/libltfs/xattr.h b/src/libltfs/xattr.h index 030fdc6d..63cd1070 100644 --- a/src/libltfs/xattr.h +++ b/src/libltfs/xattr.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2021 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -66,6 +70,11 @@ extern "C" { #define LTFS_PRIVATE_PREFIX "ltfs." +/* Explicitly export functions as utility */ +int xattr_get_string(const char *val, char **outval, const char *msg); +int xattr_get_u64(uint64_t val, char **outval, const char *msg); + +/* Official functions to export */ int xattr_set(struct dentry *d, const char *name, const char *value, size_t size, int flags, struct ltfs_volume *vol); int xattr_get(struct dentry *d, const char *name, char *value, size_t size, @@ -73,14 +82,11 @@ int xattr_get(struct dentry *d, const char *name, char *value, size_t size, int xattr_list(struct dentry *d, char *list, size_t size, struct ltfs_volume *vol); int xattr_remove(struct dentry *d, const char *name, struct ltfs_volume *vol); -int _xattr_get_string(const char *val, char **outval, const char *msg); -int _xattr_get_u64(uint64_t val, char **outval, const char *msg); - /** For internal use only */ int xattr_do_set(struct dentry *d, const char *name, const char *value, size_t size, struct xattr_info *xattr); int xattr_do_remove(struct dentry *d, const char *name, bool force, struct ltfs_volume *vol); -const char *_xattr_strip_name(const char *name); +const char *xattr_strip_name(const char *name); int xattr_set_mountpoint_length(struct dentry *d, const char* value, size_t size); #ifdef __cplusplus diff --git a/src/libltfs/xml.h b/src/libltfs/xml.h index 14f1eab7..4701242e 100644 --- a/src/libltfs/xml.h +++ b/src/libltfs/xml.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -50,7 +50,7 @@ ** ** Atsushi Abe ** IBM Tokyo Lab., Japan -** piste@jp.ibm.com +** piste@jp.ibm.com ** ************************************************************************************* */ @@ -115,7 +115,7 @@ int xml_format_time(struct ltfs_timespec t, char** out); /* standard parser variables */ #define declare_parser(toptag) \ const char *name, *parent_tag = (toptag); \ - int i, type, empty; + int i, type, empty, ret; #define declare_parser_vars_noloop(toptag) \ const char *name, *value, *parent_tag = (toptag); \ @@ -123,11 +123,11 @@ int xml_format_time(struct ltfs_timespec t, char** out); #define declare_parser_vars(toptag) \ const char *name, *value, *parent_tag = (toptag); \ - int i, type, empty; + int i, type, empty, ret; #define declare_parser_vars_symlinknode(toptag) \ const char *name, *parent_tag = (toptag); \ - int type; + int type, ret; #define declare_parser_vars_symlink(toptag) \ const char *name, *parent_tag = (toptag); \ @@ -149,123 +149,143 @@ int xml_format_time(struct ltfs_timespec t, char** out); * NOTE: in order for break to work correctly, this macro is not wrapped in a do { ... } while (0) * loop. So be careful when using it! */ #define get_next_tag() \ - if (xml_next_tag(reader, parent_tag, &name, &type) < 0) \ - return -1; \ - if (type == XML_ELEMENT_DECL) \ + ret = xml_next_tag(reader, parent_tag, &name, &type); \ + if (ret < 0) \ + return ret; \ + if (type == XML_ELEMENT_DECL) \ break /* check standard tracking array for required tags which are not present. */ -#define check_required_tags() do { \ - for (i=0; i. */ -#define assert_not_empty() do { \ - empty = xmlTextReaderIsEmptyElement(reader); \ - if (empty < 0) { \ - ltfsmsg(LTFS_ERR, 17003E); \ - return -1; \ - } else if (empty > 0) { \ - ltfsmsg(LTFS_ERR, 17004E, name); \ - return -1; \ - } \ -} while (0) +#define assert_not_empty() \ + do { \ + empty = xmlTextReaderIsEmptyElement(reader); \ + if (empty < 0) { \ + ltfsmsg(LTFS_ERR, 17003E); \ + return -LTFS_XML_EMPTY_UNKNOWN; \ + } else if (empty > 0) { \ + ltfsmsg(LTFS_ERR, 17004E, name); \ + return -LTFS_XML_EMPTY; \ + } \ + } while (0) /* check whether a tag is empty. */ -#define check_empty() do { \ - empty = xmlTextReaderIsEmptyElement(reader); \ - if (empty < 0) { \ - ltfsmsg(LTFS_ERR, 17003E); \ - return -1; \ - } \ -} while (0) +#define check_empty() \ + do { \ + empty = xmlTextReaderIsEmptyElement(reader); \ + if (empty < 0) { \ + ltfsmsg(LTFS_ERR, 17003E); \ + return -LTFS_XML_EMPTY_UNKNOWN; \ + } \ + } while (0) /* consume the end of a tag, failing if there's extra content */ -#define check_tag_end(tagname) do { \ - if (xml_next_tag(reader, (tagname), &name, &type) < 0 || type != XML_ELEMENT_DECL) { \ - ltfsmsg(LTFS_ERR, 17005E, (tagname)); \ - return -1; \ - } \ -} while (0) +#define check_tag_end(tagname) \ + do { \ + if (xml_next_tag(reader, (tagname), &name, &type) < 0 || type != XML_ELEMENT_DECL) { \ + ltfsmsg(LTFS_ERR, 17005E, (tagname)); \ + return -LTFS_XML_OPEN_TAG; \ + } \ + } while (0) /* get text from a tag, failing if the tag is empty (like ) or contains * the empty string (like ). if successful, it * reads the text into "value". It does not consume the remainder of the tag. */ -#define get_tag_text() do { \ - assert_not_empty(); \ - if (xml_scan_text(reader, &value) < 0) \ - return -1; \ - if (strlen(value) == 0) { \ - ltfsmsg(LTFS_ERR, 17004E, name); \ - return -1; \ - } \ -} while (0) - -#define get_tag_text_allow_zero_length() do { \ - assert_not_empty(); \ - if (xml_scan_text(reader, &value) < 0) \ - return -1; \ -} while (0) +#define get_tag_text() \ + do { \ + assert_not_empty(); \ + ret = xml_scan_text(reader, &value); \ + if (ret < 0) \ + return ret; \ + if (strlen(value) == 0) { \ + ltfsmsg(LTFS_ERR, 17004E, name); \ + return -LTFS_XML_EMPTY; \ + } \ + } while (0) + +#define get_tag_text_allow_zero_length() \ + do { \ + assert_not_empty(); \ + ret = xml_scan_text(reader, &value); \ + if (ret < 0) \ + return ret; \ + } while (0) /* get text from a tag. if successful, it reads the text into "value". * It does not consume the remainder of the tag. */ -#define get_tag_text_allow_empty() do { \ - if (xml_scan_text(reader, &value) < 0) \ - return -1; \ -} while (0) +#define get_tag_text_allow_empty() \ + do { \ + ret = xml_scan_text(reader, &value); \ + if (ret < 0) { \ + return ret; \ + } \ + } while (0) /* issue a warning that the tag is unrecognized and will be ignored. */ -#define ignore_unrecognized_tag() do { \ - ltfsmsg(LTFS_WARN, 17006W, name, parent_tag); \ - if (xml_skip_tag(reader) < 0) \ - return -1; \ -} while (0) +#define ignore_unrecognized_tag() \ + do { \ + ltfsmsg(LTFS_WARN, 17006W, name, parent_tag); \ + if (xml_skip_tag(reader) < 0) \ + return -LTFS_XML_SKIP_FAIL; \ + } while (0) /* store a tag in a list of unrecognized tags, to be written back to tape later */ -#define preserve_unrecognized_tag(structure) do { \ - if (xml_save_tag(reader, &(structure)->tag_count, &(structure)->preserved_tags) < 0) \ - return -1; \ - if (xml_skip_tag(reader) < 0) \ - return -1; \ -} while (0) +#define preserve_unrecognized_tag(structure) \ + do { \ + ret = xml_save_tag(reader, &(structure)->tag_count, &(structure)->preserved_tags); \ + if (ret < 0) \ + return -LTFS_XML_SAVE_FAIL; \ + if (xml_skip_tag(reader) < 0) \ + return -LTFS_XML_SKIP_FAIL; \ + } while (0) /** * This structure is used to store state data when reading XML directly from tape using * the libxml2 I/O callback method. */ struct xml_input_tape { - struct ltfs_volume *vol; /**< LTFS volume to read */ - uint64_t current_pos; /**< Current block position of the drive. */ - uint64_t eod_pos; /**< EOD position of the current partition. */ - bool saw_small_block; /**< Have we seen a small block yet? */ - bool saw_file_mark; /**< If we saw a small blilock, was it a file mark? */ - char *buf; /**< 1-block input buffer. */ - uint32_t buf_size; /**< Input buffer size. */ - uint32_t buf_start; /**< Offset of first valid byte in input buffer. */ - uint32_t buf_used; /**< Current input buffer usage. */ + struct ltfs_volume *vol; /**< LTFS volume to read */ + int err_code; /**< Error code from tape backend */ + uint64_t current_pos; /**< Current block position of the drive. */ + uint64_t eod_pos; /**< EOD position of the current partition. */ + bool saw_small_block; /**< Have we seen a small block yet? */ + bool saw_file_mark; /**< If we saw a small blilock, was it a file mark? */ + int fd; /**< File Descriptor for index cache if fd > 0 */ + int errno_fd; /**< errno from the index cache */ + char *buf; /**< 1-block input buffer. */ + uint32_t buf_size; /**< Input buffer size. */ + uint32_t buf_start; /**< Offset of first valid byte in input buffer. */ + uint32_t buf_used; /**< Current input buffer usage. */ }; int xml_input_tape_read_callback(void *context, char *buffer, int len); int xml_input_tape_close_callback(void *context); diff --git a/src/libltfs/xml_libltfs.h b/src/libltfs/xml_libltfs.h index b50a3800..b2e85f70 100644 --- a/src/libltfs/xml_libltfs.h +++ b/src/libltfs/xml_libltfs.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -36,7 +36,7 @@ ** ** COMPONENT NAME: IBM Linear Tape File System ** -** FILE NAME: xml.h +** FILE NAME: xml_libltfs.h ** ** DESCRIPTION: Prototypes for XML read/write functions. ** @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -79,13 +83,13 @@ xmlBufferPtr xml_make_label(const char *creator, tape_partition_t partition, xmlBufferPtr xml_make_schema(const char *creator, const struct ltfs_index *idx); int xml_schema_to_file(const char *filename, const char *creator, const char *reason, const struct ltfs_index *idx); -int xml_schema_to_tape(char *reason, struct ltfs_volume *vol); +int xml_schema_to_tape(char *reason, int type, struct ltfs_volume *vol); /* Functions for reading XML files. See xml_reader_libltfs.c */ int xml_label_from_file(const char *filename, struct ltfs_label *label); int xml_label_from_mem(const char *buf, int buf_size, struct ltfs_label *label); int xml_schema_from_file(const char *filename, struct ltfs_index *idx, struct ltfs_volume *vol); -int xml_schema_from_tape(uint64_t eod_pos, struct ltfs_volume *vol); +int xml_schema_from_tape(uint64_t eod_pos, bool skip_dir, struct ltfs_volume *vol); int xml_extent_symlink_info_from_file(const char *filename, struct dentry *d); #endif /* __xml_libltfs_h */ diff --git a/src/libltfs/xml_reader.c b/src/libltfs/xml_reader.c index 270c223c..80e029a4 100644 --- a/src/libltfs/xml_reader.c +++ b/src/libltfs/xml_reader.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -73,7 +77,7 @@ int xml_scan_text(xmlTextReaderPtr reader, const char **value) int type; if (xml_reader_read(reader) < 0) - return -1; + return -LTFS_XML_READ_FAIL; type = xmlTextReaderNodeType(reader); if (type == XML_ELEMENT_DECL) @@ -84,11 +88,11 @@ int xml_scan_text(xmlTextReaderPtr reader, const char **value) *value = (const char *)xmlTextReaderConstValue(reader); if (!(*value)) { ltfsmsg(LTFS_ERR, 17035E); - return -1; + return -LTFS_XML_CONST_FAIL; } } else { ltfsmsg(LTFS_ERR, 17036E, type); - return -1; + return -LTFS_XML_WRONG_NODE; } return 0; @@ -108,9 +112,11 @@ int xml_scan_text(xmlTextReaderPtr reader, const char **value) int xml_next_tag(xmlTextReaderPtr reader, const char *containing_name, const char **name, int *type) { + int ret; do { - if (xml_reader_read(reader) < 0) - return -1; + ret = xml_reader_read(reader); + if (ret < 0) + return ret; *name = (const char *)xmlTextReaderConstName(reader); *type = xmlTextReaderNodeType(reader); } while (strcmp(*name, containing_name) && (*type) != XML_ELEMENT_NODE); @@ -249,17 +255,19 @@ int xml_reader_read(xmlTextReaderPtr reader) int ret = xmlTextReaderRead(reader); if (ret < 0) { ltfsmsg(LTFS_ERR, 17037E); - return -1; + return -LTFS_XML_READ_FAIL; } else if (ret == 0) { ltfsmsg(LTFS_ERR, 17038E); - return -1; + return -LTFS_XML_UNEXPECTED_EOF; } return 0; } /** - * Parse a UUID from the tape into a provided buffer, converting to lower-case. The output buffer - * must be at least 37 bytes. + * Parse a UUID from the tape into a provided buffer, converting to lower-case. + * The output buffer must be at least 37. + * + * @return 0 on success or -1 on error (intentionally -1) */ int xml_parse_uuid(char *out_val, const char *val) { @@ -385,6 +393,8 @@ int xml_parse_ll(long long *out_val, const char *val) * Parse a positive base-10 unsigned long long from a string. * This function does not print an error message because it usually doesn't * have enough information to say something helpful. + * + * @return 0 on success or -1 on error (intentionally -1) */ int xml_parse_ull(unsigned long long *out_val, const char *val) { @@ -409,6 +419,8 @@ int xml_parse_ull(unsigned long long *out_val, const char *val) * Parse a positive base-16 unsigned long long from a string. * This function does not print an error message because it usually doesn't * have enough information to say something helpful. + * + * @return 0 on success or -1 on error (intentionally -1) */ int xml_parse_xll(unsigned long long *out_val, const char *val) { @@ -432,7 +444,8 @@ int xml_parse_xll(unsigned long long *out_val, const char *val) /** * Parse a boolean value from a string. Per the W3C boolean datatype, "true" and "1" return true * and "false" and "0" return false. - * @return 0 on success or a negative value on error. + * + * @return 0 on success or -1 on error (intentionally -1) */ int xml_parse_bool(bool *out_val, const char *value) { @@ -444,7 +457,7 @@ int xml_parse_bool(bool *out_val, const char *value) else if (! strcmp(value, "false") || ! strcmp(value, "0")) *out_val = false; else { - ltfsmsg(LTFS_ERR, 17032E); + ltfsmsg(LTFS_ERR, 17032E, value); return -1; } @@ -453,6 +466,8 @@ int xml_parse_bool(bool *out_val, const char *value) /** * Parse a time from the XML file into a timespec structure. + * + * @return 0 on success or -1 on error (intentionally -1) */ int xml_parse_time(bool msg, const char *fmt_time, struct ltfs_timespec *rawtime) { @@ -496,7 +511,7 @@ int xml_input_tape_read_callback(void *context, char *buffer, int len) struct xml_input_tape *ctx = context; ssize_t nread, nr2; char *buf2; - int bytes_saved, bytes_remaining; + int bytes_saved, bytes_remaining, ret_sp, ret_fd; if (len == 0) return 0; @@ -530,11 +545,24 @@ int xml_input_tape_read_callback(void *context, char *buffer, int len) /* Try to read a block into the buffer. */ nread = tape_read(ctx->vol->device, ctx->buf, ctx->buf_size, false, - ctx->vol->kmi_handle); + ctx->vol->kmi_handle); ++ctx->current_pos; + + /* Write down the data read to the read cache */ + if (ctx->fd > 0 && nread > 0) { + ret_fd = write(ctx->fd, ctx->buf, nread); + if (ret_fd < 0) { + ltfsmsg(LTFS_ERR, 17244E, (int)errno); + ctx->errno_fd = -LTFS_CACHE_IO; + return -1; + } + } + + /* Condition check of data read */ if (nread < 0) { /* We know we're not at EOD, so read errors are unexpected. */ ltfsmsg(LTFS_ERR, 17039E, (int)nread); + ctx->err_code = nread; return -1; } else if ((size_t) nread < ctx->buf_size) { /* Caught a small read. If this is a file mark, position before it. If @@ -542,16 +570,19 @@ int xml_input_tape_read_callback(void *context, char *buffer, int len) ctx->saw_small_block = true; if (nread == 0) { ctx->saw_file_mark = true; - if (tape_spacefm(ctx->vol->device, -1) < 0) { + ret_sp = tape_spacefm(ctx->vol->device, -1); + if (ret_sp < 0) { ltfsmsg(LTFS_ERR, 17040E); + ctx->err_code = ret_sp; return -1; } } else if (ctx->eod_pos == 0 || (ctx->eod_pos > 0 && ctx->current_pos < ctx->eod_pos)) { /* Look for a trailing file mark. */ buf2 = malloc(ctx->vol->label->blocksize); - if (! buf2) { + if (!buf2) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); + ctx->err_code = -LTFS_NO_MEMORY; return -1; } nr2 = tape_read(ctx->vol->device, buf2, ctx->vol->label->blocksize, false, @@ -560,11 +591,14 @@ int xml_input_tape_read_callback(void *context, char *buffer, int len) errno = 0; /* Clear errno because some OSs set errno in free() */ if (nr2 < 0) { /* Still not at EOD, so read errors are cause for alarm. */ ltfsmsg(LTFS_ERR, 17041E, (int)nr2); + ctx->err_code = nr2; return -1; } else if (nr2 == 0) { ctx->saw_file_mark = true; - if (tape_spacefm(ctx->vol->device, -1) < 0) { + ret_sp = tape_spacefm(ctx->vol->device, -1); + if (ret_sp < 0) { ltfsmsg(LTFS_ERR, 17040E); + ctx->err_code = ret_sp; return -1; } } @@ -589,14 +623,10 @@ int xml_input_tape_read_callback(void *context, char *buffer, int len) return len; } -/** - * Close callback for XML parser input using the libxml2 I/O routines. The input - * buffer should be empty at this point, so just free the parser context. +/** Close callback for XML parser input using the libxml2 I/O routines. */ int xml_input_tape_close_callback(void *context) { - struct xml_input_tape *ctx = context; - free(ctx->buf); - free(ctx); + /* Do nothing */ return 0; } diff --git a/src/libltfs/xml_reader_libltfs.c b/src/libltfs/xml_reader_libltfs.c index d4549b74..c4efa03f 100644 --- a/src/libltfs/xml_reader_libltfs.c +++ b/src/libltfs/xml_reader_libltfs.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -95,8 +95,9 @@ static int decode_entry_name(char **new_name, const char *name) CHECK_ARG_NULL(name, -LTFS_NULL_ARG); + /* Always, length must be shorter than original but allocate null termination space */ len = strlen(name); - tmp_name = malloc(len * sizeof(UChar)); + tmp_name = malloc((len * sizeof(UChar)) + 1); buf_decode[2] = '\0'; while (i < len) { @@ -265,6 +266,8 @@ static int _xml_parse_nametype_allow_zero_length(xmlTextReaderPtr reader, struct /** * Verify that a given string really does represent a partition (single character, a-z). + * + * @return 0 on success or -1 on error (intentionally -1) */ static int _xml_parse_partition(const char *val) { @@ -326,27 +329,29 @@ static int _xml_parser_init(xmlTextReaderPtr reader, const char *top_name, int * { const char *name, *encoding; char *value; - int type, ver; + int type, ver, ret; + + ret = xml_next_tag(reader, "", &name, &type); + if (ret < 0) + return ret; - if (xml_next_tag(reader, "", &name, &type) < 0) - return -1; if (strcmp(name, top_name)) { ltfsmsg(LTFS_ERR, 17017E, name); - return -1; + return -LTFS_XML_WRONG_TOPTAG; } /* reject this XML file if it isn't UTF-8 */ encoding = (const char *)xmlTextReaderConstEncoding(reader); if (! encoding || strcmp(encoding, "UTF-8")) { ltfsmsg(LTFS_ERR, 17018E, encoding); - return -1; + return -LTFS_XML_WRONG_ENCODING; } /* check the version attribute of the top-level tag */ value = (char *)xmlTextReaderGetAttribute(reader, BAD_CAST "version"); if (! value) { ltfsmsg(LTFS_ERR, 17019E); - return -1; + return -LTFS_XML_TOP_ATTR_FAIL; } if (_xml_parse_version(value, &ver) < 0) { ltfsmsg(LTFS_ERR, 17020E, value); @@ -367,6 +372,8 @@ static int _xml_parser_init(xmlTextReaderPtr reader, const char *top_name, int * /** * Parse a partition location from a label. + * + * @return 0 on success or -1 on error (intentionally -1) */ static int _xml_parse_label_location(xmlTextReaderPtr reader, struct ltfs_label *label) { @@ -394,6 +401,8 @@ static int _xml_parse_label_location(xmlTextReaderPtr reader, struct ltfs_label /** * Parse a partition map from an XML file, storing it in the given label structure. + * + * @return 0 on success or -1 on error (intentionally -1) */ static int _xml_parse_partition_map(xmlTextReaderPtr reader, struct ltfs_label *label) { @@ -432,15 +441,15 @@ static int _xml_parse_partition_map(xmlTextReaderPtr reader, struct ltfs_label * */ static int _xml_parse_label(xmlTextReaderPtr reader, struct ltfs_label *label) { - int ret; unsigned long long value_int; declare_parser_vars("ltfslabel"); declare_tracking_arrays(7, 0); /* start the parser: find top-level "label" tag, check version and encoding */ - if (_xml_parser_init(reader, parent_tag, &label->version, - LTFS_LABEL_VERSION_MIN, LTFS_LABEL_VERSION_MAX) < 0) - return -1; + ret = _xml_parser_init(reader, parent_tag, &label->version, + LTFS_LABEL_VERSION_MIN, LTFS_LABEL_VERSION_MAX); + if (ret < 0) + return ret; /* parse label contents */ while (true) { @@ -454,7 +463,7 @@ static int _xml_parse_label(xmlTextReaderPtr reader, struct ltfs_label *label) label->creator = strdup(value); if (! label->creator) { ltfsmsg(LTFS_ERR, 10001E, name); - return -1; + return -LTFS_NO_MEMORY; } check_tag_end("creator"); @@ -462,37 +471,44 @@ static int _xml_parse_label(xmlTextReaderPtr reader, struct ltfs_label *label) check_required_tag(1); get_tag_text(); ret = xml_parse_time(true, value, &label->format_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17268E, "formattime"); + return -LTFS_XML_WRONG_FTIME_L; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17218W, "formattime", value); check_tag_end("formattime"); } else if (! strcmp(name, "volumeuuid")) { check_required_tag(2); get_tag_text(); - if (xml_parse_uuid(label->vol_uuid, value) < 0) - return -1; + if (xml_parse_uuid(label->vol_uuid, value) < 0) { + ltfsmsg(LTFS_ERR, 17268E, "volumeuuid"); + return -LTFS_XML_WRONG_UUID; + } check_tag_end("volumeuuid"); } else if (! strcmp(name, "location")) { check_required_tag(3); assert_not_empty(); - if (_xml_parse_label_location(reader, label) < 0) - return -1; + if (_xml_parse_label_location(reader, label) < 0) { + ltfsmsg(LTFS_ERR, 17268E, "location"); + return -LTFS_XML_WRONG_LOC; + } } else if (! strcmp(name, "partitions")) { check_required_tag(4); assert_not_empty(); - if (_xml_parse_partition_map(reader, label) < 0) - return -1; + if (_xml_parse_partition_map(reader, label) < 0) { + ltfsmsg(LTFS_ERR, 17268E, "partitions"); + return -LTFS_XML_WRONG_PART_MAP; + } } else if (! strcmp(name, "blocksize")) { check_required_tag(5); get_tag_text(); if (xml_parse_ull(&value_int, value) < 0 || value_int == 0) { ltfsmsg(LTFS_ERR, 17022E, value); - return -1; + return -LTFS_XML_WRONG_BLOCKSIZE; } label->blocksize = value_int; check_tag_end("blocksize"); @@ -500,8 +516,10 @@ static int _xml_parse_label(xmlTextReaderPtr reader, struct ltfs_label *label) } else if (! strcmp(name, "compression")) { check_required_tag(6); get_tag_text(); - if (xml_parse_bool(&label->enable_compression, value) < 0) - return -1; + if (xml_parse_bool(&label->enable_compression, value) < 0) { + ltfsmsg(LTFS_ERR, 17268E, "compression"); + return -LTFS_XML_WRONG_COMP; + } check_tag_end("compression"); } else @@ -539,7 +557,7 @@ static int _xml_parse_ip_criteria(xmlTextReaderPtr reader, struct ltfs_index *id get_tag_text(); if (xml_parse_ull(&value_int, value) < 0) { ltfsmsg(LTFS_ERR, 17024E, value); - return -1; + return -LTFS_XML_WRONG_POLICY; } idx->original_criteria.max_filesize_criteria = value_int; check_tag_end("size"); @@ -570,7 +588,7 @@ static int _xml_parse_ip_criteria(xmlTextReaderPtr reader, struct ltfs_index *id if (index_criteria_dup_rules(&idx->index_criteria, &idx->original_criteria) < 0) { /* Could not duplicate index criteria rules */ ltfsmsg(LTFS_ERR, 11301E); - return -1; + return -LTFS_NO_MEMORY; } check_required_tags(); @@ -592,8 +610,9 @@ static int _xml_parse_policy(xmlTextReaderPtr reader, struct ltfs_index *idx) if (! strcmp(name, "indexpartitioncriteria")) { check_required_tag(0); assert_not_empty(); - if (_xml_parse_ip_criteria(reader, idx) < 0) - return -1; + ret = _xml_parse_ip_criteria(reader, idx); + if (ret < 0) + return ret; } else ignore_unrecognized_tag(); @@ -616,7 +635,7 @@ static int _xml_parse_one_extent(xmlTextReaderPtr reader, int idx_version, struc xt = calloc(1, sizeof(struct extent_info)); if (!xt) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); - return -1; + return -LTFS_NO_MEMORY; } while (true) { @@ -627,7 +646,7 @@ static int _xml_parse_one_extent(xmlTextReaderPtr reader, int idx_version, struc get_tag_text(); if (_xml_parse_partition(value) < 0) { free(xt); - return -1; + return -LTFS_XML_WRONG_PART; } xt->start.partition = value[0]; check_tag_end("partition"); @@ -637,7 +656,7 @@ static int _xml_parse_one_extent(xmlTextReaderPtr reader, int idx_version, struc get_tag_text(); if (xml_parse_ull(&value_int, value) < 0) { free(xt); - return -1; + return -LTFS_XML_WRONG_START_BLK; } xt->start.block = value_int; check_tag_end("startblock"); @@ -647,7 +666,7 @@ static int _xml_parse_one_extent(xmlTextReaderPtr reader, int idx_version, struc get_tag_text(); if (xml_parse_ull(&value_int, value) < 0) { free(xt); - return -1; + return -LTFS_XML_WRONG_OFFSET; } xt->byteoffset = value_int; check_tag_end("byteoffset"); @@ -657,7 +676,7 @@ static int _xml_parse_one_extent(xmlTextReaderPtr reader, int idx_version, struc get_tag_text(); if (xml_parse_ull(&value_int, value) < 0) { free(xt); - return -1; + return -LTFS_XML_WRONG_BYTE_CNT; } xt->bytecount = value_int; check_tag_end("bytecount"); @@ -667,7 +686,7 @@ static int _xml_parse_one_extent(xmlTextReaderPtr reader, int idx_version, struc get_tag_text(); if (xml_parse_ull(&value_int, value) < 0) { free(xt); - return -1; + return -LTFS_XML_WRONG_FILE_OFST; } xt->fileoffset = value_int; check_tag_end("fileoffset"); @@ -704,7 +723,7 @@ static int _xml_parse_one_extent(xmlTextReaderPtr reader, int idx_version, struc /* Overlap error */ ltfsmsg(LTFS_ERR, 17097E); free(xt); - return -1; + return -LTFS_XML_EXT_OVERLAP; } } if (! xt_used) @@ -735,8 +754,9 @@ static int _xml_parse_extents(xmlTextReaderPtr reader, int idx_version, struct d if (! strcmp(name, "extent")) { assert_not_empty(); - if (_xml_parse_one_extent(reader, idx_version, d) < 0) - return -1; + ret = _xml_parse_one_extent(reader, idx_version, d); + if (ret < 0) + return ret; } else ignore_unrecognized_tag(); @@ -752,14 +772,14 @@ static int _xml_parse_extents(xmlTextReaderPtr reader, int idx_version, struct d static int _xml_parse_one_xattr(xmlTextReaderPtr reader, struct dentry *d) { char *xattr_type; - struct xattr_info *xattr; + struct xattr_info *xattr = NULL; declare_parser_vars("xattr"); declare_tracking_arrays(2, 0); xattr = calloc(1, sizeof(struct xattr_info)); if (! xattr) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); - return -1; + return -LTFS_NO_MEMORY; } while (true) { @@ -769,8 +789,12 @@ static int _xml_parse_one_xattr(xmlTextReaderPtr reader, struct dentry *d) check_required_tag(0); /* Allow slash in xattr key */ - if (_xml_parse_nametype(reader, &xattr->key, true) < 0) + ret = _xml_parse_nametype(reader, &xattr->key, true); + if (ret < 0) { + ltfsmsg(LTFS_WARN, 17269W, d->name.name); free(xattr); + xattr = NULL; + } check_tag_end("key"); @@ -781,57 +805,61 @@ static int _xml_parse_one_xattr(xmlTextReaderPtr reader, struct dentry *d) if (xattr_type && strcmp(xattr_type, "text") && strcmp(xattr_type, "base64")) { ltfsmsg(LTFS_ERR, 17027E, xattr_type); free(xattr); - return -1; + return -LTFS_XML_XATTR_TYPE; } check_empty(); - if (empty == 0) { - if (xml_scan_text(reader, &value) < 0) { - free(xattr->key.name); - free(xattr); - return -1; - } - if (! xattr_type || ! strcmp(xattr_type, "text")) { - xattr->value = strdup(value); - if (! xattr->value) { - ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); + if (xattr) { + if (empty == 0) { + ret = xml_scan_text(reader, &value); + if (ret < 0) { free(xattr->key.name); free(xattr); - return -1; + return ret; } - xattr->size = strlen(value); - } else { /* base64 */ - xattr->size = base64_decode( - (const unsigned char *)value, - strlen(value), - (unsigned char **)(&xattr->value)); - if (xattr->size == 0) { - ltfsmsg(LTFS_ERR, 17028E); - free(xattr->key.name); - free(xattr); - return -1; + + if (! xattr_type || ! strcmp(xattr_type, "text")) { + xattr->value = strdup(value); + if (! xattr->value) { + ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); + free(xattr->key.name); + free(xattr); + return -LTFS_NO_MEMORY; + } + xattr->size = strlen(value); + } else { /* base64 */ + xattr->size = base64_decode((const unsigned char *)value, strlen(value), + (unsigned char **)(&xattr->value)); + if (xattr->size == 0) { + ltfsmsg(LTFS_ERR, 17028E); + free(xattr->key.name); + free(xattr); + return -LTFS_XML_XATTR_SIZE; + } } + if (strlen(value) > 0) + check_tag_end("value"); + } else { + xattr->value = NULL; + xattr->size = 0; } - if (strlen(value) > 0) - check_tag_end("value"); - } else { - xattr->value = NULL; - xattr->size = 0; } free(xattr_type); - } else ignore_unrecognized_tag(); } check_required_tags(); - TAILQ_INSERT_TAIL(&d->xattrlist, xattr, list); - if (!strcmp(xattr->key.name, "ltfs.vendor.IBM.immutable") && !strcmp(xattr->value, "1") ) { - d->is_immutable = true; - } - if (!strcmp(xattr->key.name, "ltfs.vendor.IBM.appendonly") && !strcmp(xattr->value, "1") ) { - d->is_appendonly = true; + if (xattr) { + TAILQ_INSERT_TAIL(&d->xattrlist, xattr, list); + + if (!strcmp(xattr->key.name, "ltfs.vendor.IBM.immutable") && !strcmp(xattr->value, "1") ) { + d->is_immutable = true; + } + if (!strcmp(xattr->key.name, "ltfs.vendor.IBM.appendonly") && !strcmp(xattr->value, "1") ) { + d->is_appendonly = true; + } } return 0; @@ -850,8 +878,9 @@ static int _xml_parse_xattrs(xmlTextReaderPtr reader, struct dentry *d) if (! strcmp(name, "xattr")) { assert_not_empty(); - if (_xml_parse_one_xattr(reader, d) < 0) - return -1; + ret = _xml_parse_one_xattr(reader, d); + if (ret < 0) + return ret; } else ignore_unrecognized_tag(); @@ -863,12 +892,13 @@ static int _xml_parse_xattrs(xmlTextReaderPtr reader, struct dentry *d) /** * Parse a tape offset (a partition tag and a startblock tag) from an XML source. + * * @param reader the XML source * @param tag name of the tag containing the tape position. This function will read to the end * of that tag, so it is not suitable for reading an extent list (which has other * children that need to be parsed). * @param pos pointer to a structure where the offset will be stored - * @return 0 on success or a negative value on error + * @return 0 on success or -1 on error (intentionally -1) */ static int _xml_parse_tapepos(xmlTextReaderPtr reader, const char *tag, struct tape_offset *pos) { @@ -914,7 +944,7 @@ static int _xml_save_symlink_conflict( struct ltfs_index *idx, struct dentry *d) err_d = realloc( idx->symlink_conflict, c * sizeof(size_t)); if (! err_d) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); - return -1; + return -LTFS_NO_MEMORY; } err_d[c-1] = d; @@ -927,9 +957,9 @@ static int _xml_save_symlink_conflict( struct ltfs_index *idx, struct dentry *d) /** * Parse a file into the given directory. */ -static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, struct dentry *dir, struct name_list *filename) +static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, struct dentry *dir, + struct name_list *filename) { - int ret; unsigned long long value_int; struct dentry *file; struct extent_info *xt_last; @@ -940,7 +970,7 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru file = fs_allocate_dentry(dir, NULL, NULL, false, false, false, idx); if (! file) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); - return -1; + return -LTFS_NO_MEMORY; } while (true) { @@ -962,25 +992,30 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru } else if (!strcmp(name, "length")) { check_required_tag(1); get_tag_text(); - if (xml_parse_ull(&value_int, value) < 0) - return -1; + if (xml_parse_ull(&value_int, value) < 0) { + ltfsmsg(LTFS_ERR, 17270E, "length", file->name.name); + return -LTFS_XML_WRONG_SIZE; + } file->size = value_int; check_tag_end("length"); } else if (!strcmp(name, "readonly")) { check_required_tag(2); get_tag_text(); - if (xml_parse_bool(&file->readonly, value) < 0) - return -1; + if (xml_parse_bool(&file->readonly, value) < 0) { + ltfsmsg(LTFS_ERR, 17270E, "readonly", file->name.name); + return -LTFS_XML_WRONG_RO_F; + } check_tag_end("readonly"); } else if (!strcmp(name, "modifytime")) { check_required_tag(3); get_tag_text(); ret = xml_parse_time(true, value, &file->modify_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, "modifytime", file->name.name); + return -LTFS_XML_WRONG_MTIME_F; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "modifytime", file->name.name, (unsigned long long)file->uid, value); check_tag_end("modifytime"); @@ -988,9 +1023,10 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru check_required_tag(4); get_tag_text(); ret = xml_parse_time(true, value, &file->creation_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, "creationtime", file->name.name); + return -LTFS_XML_WRONG_CRTIME_F; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "creationtime", file->name.name, (unsigned long long)file->uid, value); check_tag_end("creationtime"); @@ -999,9 +1035,10 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru check_required_tag(5); get_tag_text(); ret = xml_parse_time(true, value, &file->access_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, "accesstime", file->name.name); + return -LTFS_XML_WRONG_ATIME_F; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "accesstime", file->name.name, (unsigned long long)file->uid, value); check_tag_end("accesstime"); @@ -1009,31 +1046,44 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru check_required_tag(6); get_tag_text(); ret = xml_parse_time(true, value, &file->change_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, "changetime", file->name.name); + return -LTFS_XML_WRONG_CTIME_F; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "changetime", file->name.name, (unsigned long long)file->uid, value); check_tag_end("changetime"); } else if (!strcmp(name, "extendedattributes")) { check_optional_tag(0); check_empty(); - if (empty == 0 && _xml_parse_xattrs(reader, file) < 0) - return -1; + + if (empty == 0) { + ret = _xml_parse_xattrs(reader, file); + if(ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, "extendedattributes", file->name.name); + return ret; + } + } } else if (!strcmp(name, "extentinfo")) { check_optional_tag(1); check_empty(); - if (empty == 0 && _xml_parse_extents(reader, idx->version, file) < 0) - return -1; - else extent_flag = true; + if (empty == 0) { + ret = _xml_parse_extents(reader, idx->version, file); + if(ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, "extentinfo", file->name.name); + return ret; + } + } else extent_flag = true; } else if (!strcmp(name, "symlink")) { check_optional_tag(2); - if (_xml_parse_nametype(reader, &file->target, true) < 0) { + ret = _xml_parse_nametype(reader, &file->target, true); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, "symlink", file->name.name); free(file); - return -1; + return ret; } file->isslink = true; @@ -1055,8 +1105,10 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru } else if (idx->version >= IDX_VERSION_UID && ! strcmp(name, UID_TAGNAME)) { check_required_tag(7); get_tag_text(); - if (xml_parse_ull(&value_int, value) < 0) - return -1; + if (xml_parse_ull(&value_int, value) < 0) { + ltfsmsg(LTFS_ERR, 17270E, UID_TAGNAME, file->name.name); + return -LTFS_XML_WRONG_UID; + } file->uid = value_int; if (file->uid > idx->uid_number) idx->uid_number = file->uid; @@ -1069,9 +1121,10 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru check_required_tag(8); get_tag_text(); ret = xml_parse_time(true, value, &file->backup_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17270E, BACKUPTIME_TAGNAME, file->name.name); + return -LTFS_XML_WRONG_BTIME_F; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "backuptime", file->name.name, (unsigned long long)file->uid, value); check_tag_end(BACKUPTIME_TAGNAME); @@ -1104,19 +1157,23 @@ static int _xml_parse_file(xmlTextReaderPtr reader, struct ltfs_index *idx, stru xt_last = TAILQ_LAST(&file->extentlist, extent_struct); if (xt_last->fileoffset + xt_last->bytecount > file->size) { ltfsmsg(LTFS_ERR, 17026E); - return -1; + return -LTFS_XML_EXT_TOO_LONG; } } /* Validate UID: must be nonzero (UID 0 is reserved for the root directory) */ if (file->uid == 0) { - ltfsmsg(LTFS_ERR, 17101E); - return -1; + ltfsmsg(LTFS_ERR, 17101E, file->name.name); + return -LTFS_XML_WRONG_UID; } if ( symlink_flag && extent_flag ) { ltfsmsg(LTFS_ERR, 17180E, file->name.name); - if ( _xml_save_symlink_conflict( idx, file ) ) return -1; + ret = _xml_save_symlink_conflict(idx, file); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17271E, file->name.name); + return ret; + } } return 0; @@ -1132,7 +1189,6 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, static int _xml_parse_dir_contents(xmlTextReaderPtr reader, struct dentry *dir, struct ltfs_index *idx) { - int ret = 0; struct name_list *list = NULL, *entry_name = NULL; CHECK_ARG_NULL(dir, -LTFS_NULL_ARG); declare_parser("contents"); @@ -1173,6 +1229,7 @@ static int _xml_parse_dir_contents(xmlTextReaderPtr reader, struct dentry *dir, ignore_unrecognized_tag(); entry_name = NULL; } + if (!strcmp(name, "file") || (!strcmp(name, "directory") && !(!dir && idx->root))) { /* Make temporal hash table whose key is dentry name */ HASH_ADD_KEYPTR(hh, list, entry_name->name, strlen(entry_name->name), entry_name); @@ -1199,8 +1256,9 @@ static int _xml_parse_dir_contents(xmlTextReaderPtr reader, struct dentry *dir, The file or directory of which name contains invalid char is skipped in this step. After that update platfrom_safe_name regarding skipped file or directory as the second step. These steps make a prioritization of name mangling.*/ - if (fs_update_platform_safe_names(dir, idx, list)!=0) { - return -1; + ret = fs_update_platform_safe_names(dir, idx, list); + if (ret < 0) { + return ret; } check_required_tags(); @@ -1220,7 +1278,6 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, struct ltfs_index *idx, struct ltfs_volume *vol, struct name_list *dirname) { - int ret; unsigned long long value_int; struct dentry *dir; @@ -1252,6 +1309,7 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, if (parent) { ret = _xml_parse_nametype(reader, &dir->name, false); if (ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, "name", parent->name.name); free(dir); return ret; } @@ -1269,8 +1327,10 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, idx->volume_name.name = NULL; } else { ret = _xml_parse_nametype_allow_zero_length(reader, &idx->volume_name, false); - if (ret < 0) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, "name", "/"); return ret; + } if (idx->volume_name.name) check_tag_end("name"); @@ -1280,17 +1340,20 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, } else if (!strcmp(name, "readonly")) { check_required_tag(1); get_tag_text(); - if (xml_parse_bool(&dir->readonly, value) < 0) - return -1; + if (xml_parse_bool(&dir->readonly, value) < 0) { + ltfsmsg(LTFS_ERR, 17272E, "readonly", dir->name.name); + return -LTFS_XML_WRONG_RO_DIR; + } check_tag_end("readonly"); } else if (!strcmp(name, "modifytime")) { check_required_tag(2); get_tag_text(); ret = xml_parse_time(true, value, &dir->modify_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, "modifytime", dir->name.name); + return -LTFS_XML_WRONG_MTIME_DIR; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "updatetime", dir->name.name, (unsigned long long)dir->uid, value); check_tag_end("modifytime"); @@ -1299,9 +1362,10 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, check_required_tag(3); get_tag_text(); ret = xml_parse_time(true, value, &dir->creation_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, "creationtime", dir->name.name); + return -LTFS_XML_WRONG_CRTIME_DIR; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "creationtime", dir->name.name, (unsigned long long)dir->uid, value); check_tag_end("creationtime"); @@ -1310,9 +1374,10 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, check_required_tag(4); get_tag_text(); ret = xml_parse_time(true, value, &dir->access_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, "accesstime", dir->name.name); + return -LTFS_XML_WRONG_CTIME_DIR; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "accesstime", dir->name.name, (unsigned long long)dir->uid, value); check_tag_end("accesstime"); @@ -1321,14 +1386,15 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, check_required_tag(5); get_tag_text(); ret = xml_parse_time(true, value, &dir->change_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, "changetime", dir->name.name); + return -LTFS_XML_WRONG_CTIME_DIR; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "changetime", dir->name.name, (unsigned long long)dir->uid, value); check_tag_end("changetime"); - } else if (! strcmp(name, "contents")) { + } else if (!strcmp(name, "contents")) { check_required_tag(6); check_empty(); if (empty == 0) { @@ -1340,14 +1406,21 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, } else if (!strcmp(name, "extendedattributes")) { check_optional_tag(0); check_empty(); - if (empty == 0 && _xml_parse_xattrs(reader, dir) < 0) - return -1; + if (empty == 0) { + ret = _xml_parse_xattrs(reader, dir); + if(ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, "extendedattributes", dir->name.name); + return ret; + } + } } else if (idx->version >= IDX_VERSION_UID && ! strcmp(name, UID_TAGNAME)) { check_required_tag(7); get_tag_text(); - if (xml_parse_ull(&value_int, value) < 0) - return -1; + if (xml_parse_ull(&value_int, value) < 0) { + ltfsmsg(LTFS_ERR, 17272E, UID_TAGNAME, dir->name.name); + return -LTFS_XML_WRONG_UID; + } dir->uid = value_int; if (dir->uid > idx->uid_number) idx->uid_number = dir->uid; @@ -1362,9 +1435,10 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, check_required_tag(8); get_tag_text(); ret = xml_parse_time(true, value, &dir->backup_time); - if (ret < 0) - return -1; - else if (ret == LTFS_TIME_OUT_OF_RANGE) + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17272E, BACKUPTIME_TAGNAME, dir->name.name); + return -LTFS_XML_WRONG_BTIME_DIR; + } else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17220W, "backuptime", dir->name.name, (unsigned long long)dir->uid, value); check_tag_end(BACKUPTIME_TAGNAME); @@ -1398,14 +1472,14 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, /* Validate UID: root directory must have uid==1, other dentries must have nonzero UID */ /* TODO: would be nice to verify that there are no UID conflicts */ if (parent && dir->uid == 1) { - ltfsmsg(LTFS_ERR, 17101E); - return -1; + ltfsmsg(LTFS_ERR, 17101E, dir->name.name); + return -LTFS_XML_INVALID_UID; } else if (! parent && dir->uid != 1) { ltfsmsg(LTFS_ERR, 17100E); - return -1; + return -LTFS_XML_INVALID_UID; } else if (dir->uid == 0) { ltfsmsg(LTFS_ERR, 17106E); - return -1; + return -LTFS_XML_INVALID_UID; } return 0; @@ -1416,12 +1490,13 @@ static int _xml_parse_dirtree(xmlTextReaderPtr reader, struct dentry *parent, * with the nodes found during the scanning. * @param reader Source of XML data * @param idx LTFS index + * @param skip_dir skip parsing directory and file * @param vol LTFS volume to which the index belongs. May be NULL. * @return 0 on success or a negative value on error. */ -static int _xml_parse_schema(xmlTextReaderPtr reader, struct ltfs_index *idx, struct ltfs_volume *vol) +static int _xml_parse_schema(xmlTextReaderPtr reader, bool skip_dir, + struct ltfs_index *idx, struct ltfs_volume *vol) { - int ret; unsigned long long value_int; declare_parser_vars("ltfsindex"); declare_tracking_arrays(8, 4); @@ -1468,7 +1543,7 @@ static int _xml_parse_schema(xmlTextReaderPtr reader, struct ltfs_index *idx, st idx->creator = strdup(value); if (! idx->creator) { ltfsmsg(LTFS_ERR, 10001E, name); - return -1; + return -LTFS_NO_MEMORY; } check_tag_end("creator"); @@ -1476,7 +1551,7 @@ static int _xml_parse_schema(xmlTextReaderPtr reader, struct ltfs_index *idx, st check_required_tag(1); get_tag_text(); if (xml_parse_uuid(idx->vol_uuid, value) < 0) - return -1; + return -LTFS_XML_WRONG_UUID; check_tag_end("volumeuuid"); } else if (! strcmp(name, "generationnumber")) { @@ -1484,7 +1559,7 @@ static int _xml_parse_schema(xmlTextReaderPtr reader, struct ltfs_index *idx, st get_tag_text(); if (xml_parse_ull(&value_int, value) < 0) { ltfsmsg(LTFS_ERR, 17023E, value); - return -1; + return -LTFS_XML_WRONG_GEN; } idx->generation = value_int; check_tag_end("generationnumber"); @@ -1494,7 +1569,7 @@ static int _xml_parse_schema(xmlTextReaderPtr reader, struct ltfs_index *idx, st get_tag_text(); ret = xml_parse_time(true, value, &idx->mod_time); if (ret < 0) - return -1; + return -LTFS_XML_WRONG_UTIME; else if (ret == LTFS_TIME_OUT_OF_RANGE) ltfsmsg(LTFS_WARN, 17219W, "updatetime", value); @@ -1504,47 +1579,53 @@ static int _xml_parse_schema(xmlTextReaderPtr reader, struct ltfs_index *idx, st check_required_tag(4); assert_not_empty(); if (_xml_parse_tapepos(reader, "location", &idx->selfptr) < 0) - return -1; + return -LTFS_XML_WRONG_LOC; } else if (! strcmp(name, "allowpolicyupdate")) { check_required_tag(5); get_tag_text(); if (xml_parse_bool(&idx->criteria_allow_update, value) < 0) - return -1; + return -LTFS_XML_WRONG_PA; check_tag_end("allowpolicyupdate"); } else if (! strcmp(name, "directory")) { check_required_tag(6); assert_not_empty(); - ret = _xml_parse_dirtree(reader, NULL, idx, vol, NULL); - if (ret < 0) - return ret; + if (skip_dir) { + xml_skip_tag(reader); + } else { + ret = _xml_parse_dirtree(reader, NULL, idx, vol, NULL); + if (ret < 0) + return ret; + } } else if (! strcmp(name, "previousgenerationlocation")) { check_optional_tag(0); assert_not_empty(); if (_xml_parse_tapepos(reader, "previousgenerationlocation", &idx->backptr) < 0) - return -1; + return -LTFS_XML_WRONG_LOC_PREV; } else if (! strcmp(name, "dataplacementpolicy")) { check_optional_tag(1); assert_not_empty(); - if (_xml_parse_policy(reader, idx) < 0) - return -1; + ret = _xml_parse_policy(reader, idx); + if (ret < 0) + return ret; } else if (! strcmp(name, "comment")) { check_optional_tag(2); get_tag_text(); if (strlen(value) > INDEX_MAX_COMMENT_LEN) { ltfsmsg(LTFS_ERR, 17094E); - return -1; + return -LTFS_XML_TOO_LONG_COMMENT; } idx->commit_message = strdup(value); if (! idx->commit_message) { ltfsmsg(LTFS_ERR, 10001E, "_xml_parse_schema: index comment"); - return -1; + return -LTFS_NO_MEMORY; } check_tag_end("comment"); + } else if (! strcmp(name, "volumelockstate")) { check_optional_tag(3); get_tag_text(); @@ -1557,14 +1638,16 @@ static int _xml_parse_schema(xmlTextReaderPtr reader, struct ltfs_index *idx, st idx->vollock = PERMLOCKED_MAM; } check_tag_end("volumelockstate"); + } else if (idx->version >= IDX_VERSION_UID && ! strcmp(name, NEXTUID_TAGNAME)) { check_required_tag(7); get_tag_text(); if (xml_parse_ull(&value_int, value) < 0) - return -1; + return -LTFS_XML_WRONG_NEXT; if (value_int > idx->uid_number) idx->uid_number = value_int; check_tag_end(NEXTUID_TAGNAME); + } else if (! strcmp(name, NEXTUID_TAGNAME)) { ignore_unrecognized_tag(); @@ -1598,8 +1681,9 @@ static int _xml_parse_symlink_target(xmlTextReaderPtr reader, int idx_version, s if (! strcmp(name, "target")) { d->isslink = true; - if (_xml_parse_nametype(reader, &d->target, true) < 0) - return -1; + ret = _xml_parse_nametype(reader, &d->target, true); + if (ret < 0) + return ret; } else ignore_unrecognized_tag(); } @@ -1737,7 +1821,8 @@ int xml_label_from_mem(const char *buf, int buf_size, struct ltfs_label *label) ret = _xml_parse_label(reader, label); if (ret < 0) { - ltfsmsg(LTFS_ERR, 17010E); + /* TODO: Update the message */ + ltfsmsg(LTFS_ERR, 17010E, ret); ret = -LTFS_LABEL_INVALID; } xmlFreeTextReader(reader); @@ -1750,7 +1835,7 @@ int xml_label_from_mem(const char *buf, int buf_size, struct ltfs_label *label) * with the nodes found during the scanning. * @param filename XML input file. * @param idx LTFS index. - * @param vol LTFS volume to which the index belongs. May be NULL. + * @param vol LTFS volume to which the index belongs. Can be NULL. * @return 0 on success or a negative value on error. */ int xml_schema_from_file(const char *filename, struct ltfs_index *idx, struct ltfs_volume *vol) @@ -1765,16 +1850,16 @@ int xml_schema_from_file(const char *filename, struct ltfs_index *idx, struct lt reader = xmlReaderForFile(filename, NULL, XML_PARSE_NOERROR | XML_PARSE_NOWARNING | XML_PARSE_HUGE); if (! reader) { ltfsmsg(LTFS_ERR, 17011E, filename); - return -1; + return -LTFS_FILE_ERR; } /* Workaround for old libxml2 version on OS X 10.5: the method used to preserve * unknown tags modifies the behavior of xmlFreeTextReader so that an additional * xmlDocFree call is required to free all memory. */ doc = xmlTextReaderCurrentDoc(reader); - ret = _xml_parse_schema(reader, idx, vol); + ret = _xml_parse_schema(reader, false, idx, vol); if (ret < 0) - ltfsmsg(LTFS_ERR, 17012E, filename); + ltfsmsg(LTFS_ERR, 17012E, filename, ret); if (doc) xmlFreeDoc(doc); xmlFreeTextReader(reader); @@ -1795,13 +1880,14 @@ int xml_schema_from_file(const char *filename, struct ltfs_index *idx, struct lt * the file mark. * @param eod_pos EOD block position for the current partition, or 0 to assume EOD will not be * encountered during parsing. + * @param skip_dir skip parsing directory * @param vol LTFS volume. * @return 0 on success, 1 if parsing succeeded but no file mark was encountered, * or a negative value on error. */ -int xml_schema_from_tape(uint64_t eod_pos, struct ltfs_volume *vol) +int xml_schema_from_tape(uint64_t eod_pos, bool skip_dir, struct ltfs_volume *vol) { - int ret; + int ret, bk = -1; struct tc_position current_pos; struct xml_input_tape *ctx; xmlParserInputBufferPtr read_buf; @@ -1828,14 +1914,21 @@ int xml_schema_from_tape(uint64_t eod_pos, struct ltfs_volume *vol) free(ctx); return -LTFS_NO_MEMORY; } - ctx->vol = vol; - ctx->current_pos = current_pos.block; - ctx->eod_pos = eod_pos; + + ctx->fd = -1; + ctx->errno_fd = 0; + if (vol->index_cache_path_r) + xml_acquire_file_lock(vol->index_cache_path_r, &ctx->fd, &bk, true); + + ctx->vol = vol; + ctx->err_code = 0; + ctx->current_pos = current_pos.block; + ctx->eod_pos = eod_pos; ctx->saw_small_block = false; - ctx->saw_file_mark = false; - ctx->buf_size = vol->label->blocksize; - ctx->buf_start = 0; - ctx->buf_used = 0; + ctx->saw_file_mark = false; + ctx->buf_size = vol->label->blocksize; + ctx->buf_start = 0; + ctx->buf_used = 0; /* Create input buffer pointer. */ read_buf = xmlParserInputBufferCreateIO(xml_input_tape_read_callback, @@ -1843,6 +1936,8 @@ int xml_schema_from_tape(uint64_t eod_pos, struct ltfs_volume *vol) ctx, XML_CHAR_ENCODING_NONE); if (! read_buf) { ltfsmsg(LTFS_ERR, 17014E); + if (ctx->fd >= 0) + xml_release_file_lock(vol->index_cache_path_r, ctx->fd, bk, false); free(ctx->buf); free(ctx); return -LTFS_LIBXML2_FAILURE; @@ -1853,6 +1948,10 @@ int xml_schema_from_tape(uint64_t eod_pos, struct ltfs_volume *vol) if (! reader) { ltfsmsg(LTFS_ERR, 17015E); xmlFreeParserInputBuffer(read_buf); + if (ctx->fd >= 0) + xml_release_file_lock(vol->index_cache_path_r, ctx->fd, bk, false); + free(ctx->buf); + free(ctx); return -LTFS_LIBXML2_FAILURE; } @@ -1863,6 +1962,10 @@ int xml_schema_from_tape(uint64_t eod_pos, struct ltfs_volume *vol) ltfsmsg(LTFS_ERR, 17015E); xmlFreeTextReader(reader); xmlFreeParserInputBuffer(read_buf); + if (ctx->fd >= 0) + xml_release_file_lock(vol->index_cache_path_r, ctx->fd, bk, false); + free(ctx->buf); + free(ctx); return -LTFS_LIBXML2_FAILURE; } #endif @@ -1872,22 +1975,37 @@ int xml_schema_from_tape(uint64_t eod_pos, struct ltfs_volume *vol) doc = xmlTextReaderCurrentDoc(reader); /* Generate the Index. */ - ret = _xml_parse_schema(reader, vol->index, vol); + ret = _xml_parse_schema(reader, skip_dir, vol->index, vol); + if (ctx->err_code < 0) { + /* Error happens while reading tape */ + ltfsmsg(LTFS_ERR, 17273E, ctx->err_code); + ret = ctx->err_code; + } if (ret < 0) { - ltfsmsg(LTFS_ERR, 17016E); + ltfsmsg(LTFS_ERR, 17016E, ret); if (ret == -1) { - /* TODO: Need to return more descriptive error codes */ + /* Unexpected error code, we need to investigate */ + ltfsmsg(LTFS_WARN, 17274W, ret); ret = -LTFS_INDEX_INVALID; } } else if (ret == 0) { - if( ! ctx->saw_file_mark) - ret = 1; + if(!ctx->saw_file_mark) { + /* Return positive value intentionally, for recovering later */ + ret = LTFS_NO_TRAIL_FM; + } } + if (doc) xmlFreeDoc(doc); xmlFreeTextReader(reader); xmlFreeParserInputBuffer(read_buf); + if (ctx->fd >= 0) + xml_release_file_lock(vol->index_cache_path_r, ctx->fd, bk, false); + + free(ctx->buf); + free(ctx); + #ifdef DEBUG /* dump the tree if it isn't too large */ if (ret >= 0 && vol->index->file_count < 1000) diff --git a/src/libltfs/xml_writer.c b/src/libltfs/xml_writer.c index a0554dab..53ae0de0 100644 --- a/src/libltfs/xml_writer.c +++ b/src/libltfs/xml_writer.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -61,11 +65,6 @@ #include "pathname.h" #include "arch/time_internal.h" -/* O_BINARY is defined only in MinGW */ -#ifndef O_BINARY -#define O_BINARY 0 -#endif - /** * Format a raw timespec structure for the XML file. */ @@ -321,7 +320,7 @@ int xml_acquire_file_lock(const char *file, int *fd, int *bk_fd, bool is_write) /* Open specified file to lock */ *fd = open(file, O_RDWR | O_CREAT | O_BINARY, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP ); if (*fd < 0) { /* Failed to open the advisory lock '%s' (%d) */ errno_save = errno; @@ -397,7 +396,7 @@ int xml_acquire_file_lock(const char *file, int *fd, int *bk_fd, bool is_write) } *bk_fd = open(backup_file, O_RDWR | O_CREAT | O_BINARY | O_TRUNC, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP ); if (*bk_fd < 0) { ltfsmsg(LTFS_ERR, 17246E, "backup file creation", errno); errno_save = errno; diff --git a/src/libltfs/xml_writer_libltfs.c b/src/libltfs/xml_writer_libltfs.c index 8dffcc4b..098b1892 100644 --- a/src/libltfs/xml_writer_libltfs.c +++ b/src/libltfs/xml_writer_libltfs.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -60,13 +60,9 @@ #include "fs.h" #include "tape.h" #include "pathname.h" +#include "inc_journal.h" #include "arch/time_internal.h" -/* O_BINARY is defined only in MinGW */ -#ifndef O_BINARY -#define O_BINARY 0 -#endif - /* Structure to control EE's file offset cache and sync file list */ struct ltfsee_cache { @@ -343,10 +339,11 @@ static int _xml_write_file(xmlTextWriterPtr writer, struct dentry *file, struct /* Write dirty file list */ if (sync_list->fp && file->dirty) { fprintf(sync_list->fp, "%s,%"PRIu64"\n", file->name.name, file->size); - file->dirty = false; sync_list->count++; } + file->dirty = false; + return 0; } @@ -400,24 +397,24 @@ static int _xml_write_dirtree(xmlTextWriterPtr writer, struct dentry *dir, HASH_ITER(hh, dir->child_list, list_ptr, list_tmp) { if (list_ptr->d->isdir) { - if (list_ptr->d->vol->index_cache_path && !strcmp(list_ptr->d->name.name, ".LTFSEE_DATA")) { - ret = asprintf(&offset_name, "%s.%s", list_ptr->d->vol->index_cache_path, "offsetcache.new"); + if (list_ptr->d->vol->index_cache_path_w && !strcmp(list_ptr->d->name.name, ".LTFSEE_DATA")) { + ret = asprintf(&offset_name, "%s.%s", list_ptr->d->vol->index_cache_path_w, "offsetcache.new"); if (ret > 0) { offset->fp = fopen(offset_name, "w"); free(offset_name); if (!offset->fp) - ltfsmsg(LTFS_WARN, 17248W, "offset cache", list_ptr->d->vol->index_cache_path); + ltfsmsg(LTFS_WARN, 17248W, "offset cache", list_ptr->d->vol->index_cache_path_w); } else - ltfsmsg(LTFS_WARN, 17247W, "offset cache", list_ptr->d->vol->index_cache_path); + ltfsmsg(LTFS_WARN, 17247W, "offset cache", list_ptr->d->vol->index_cache_path_w); - ret = asprintf(&sync_name, "%s.%s", list_ptr->d->vol->index_cache_path, "synclist.new"); + ret = asprintf(&sync_name, "%s.%s", list_ptr->d->vol->index_cache_path_w, "synclist.new"); if (ret > 0) { sync->fp = fopen(sync_name, "w"); free(sync_name); if (!sync->fp) - ltfsmsg(LTFS_WARN, 17248W, "sync list", list_ptr->d->vol->index_cache_path); + ltfsmsg(LTFS_WARN, 17248W, "sync list", list_ptr->d->vol->index_cache_path_w); } else - ltfsmsg(LTFS_WARN, 17247W, "sync list", list_ptr->d->vol->index_cache_path); + ltfsmsg(LTFS_WARN, 17247W, "sync list", list_ptr->d->vol->index_cache_path_w); } xml_mktag(_xml_write_dirtree(writer, list_ptr->d, idx, offset, sync), -1); @@ -591,6 +588,433 @@ static int _xml_write_schema(xmlTextWriterPtr writer, const char *creator, return 0; } +#ifdef FORMAT_SPEC25 +static int _xml_write_incremental_dir(xmlTextWriterPtr writer, struct dentry *dir) +{ + /* Handle R/O and timestamp if it is dirty */ + if (dir->dirty) { + xml_mktag(xmlTextWriterWriteElement( + writer, BAD_CAST "readonly", BAD_CAST (dir->readonly ? "true" : "false")), -1); + xml_mktag(_xml_write_dentry_times(writer, dir), -1); + } + + /* Handle UID in any case */ + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST UID_TAGNAME, "%"PRIu64, dir->uid), -1); + + /* Handle extended attribute if it is dirty */ + if (dir->dirty) { + xml_mktag(_xml_write_xattr(writer, dir), -1); + dir->dirty = false; + } + + return 0; +} + +static int _xml_open_incremental_dir_ent(xmlTextWriterPtr writer, struct dentry *dir) +{ + /* Handle R/O and timestamp if it is dirty */ + if (dir->dirty) { + xml_mktag(xmlTextWriterWriteElement( + writer, BAD_CAST "readonly", BAD_CAST (dir->readonly ? "true" : "false")), -1); + xml_mktag(_xml_write_dentry_times(writer, dir), -1); + } + + /* Handle UID in any case */ + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST UID_TAGNAME, "%"PRIu64, dir->uid), -1); + + /* Handle extended attribute if it is dirty */ + if (dir->dirty) { + xml_mktag(_xml_write_xattr(writer, dir), -1); + dir->dirty = false; + } + + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "contents"), -1); + + return 0; +} + +static int _xml_open_incrental_root(xmlTextWriterPtr writer, struct ltfs_volume *vol) +{ + int ret = 0; + struct ltfs_index *idx = vol->index; + struct dentry *dir = idx->root; + + /* Handle name tag */ + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "directory"), -1); + if (idx->volume_name.name) { + xml_mktag(_xml_write_nametype(writer, "name", (struct ltfs_name*)(&idx->volume_name)), -1); + } else { + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "name"), -1); + xml_mktag(xmlTextWriterEndElement(writer), -1); + } + + ret = _xml_open_incremental_dir_ent(writer, dir); + + return ret; +} + +static inline int _xml_close_incrental_root(xmlTextWriterPtr writer) +{ + xml_mktag(xmlTextWriterEndElement(writer), -1); /* close contents tag */ + xml_mktag(xmlTextWriterEndElement(writer), -1); /* close directory tag */ + return 0; +} + +static int _xml_open_incremental_dir(xmlTextWriterPtr writer, struct dentry *dir) +{ + int ret = 0; + + /* Handle name tag */ + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "directory"), -1); + xml_mktag(_xml_write_nametype(writer, "name", &dir->name), -1); + xml_mktag(xmlTextWriterEndElement(writer), -1); + + ret = _xml_open_incremental_dir_ent(writer, dir); + + return ret; +} + +static inline int _xml_close_incremental_dir(xmlTextWriterPtr writer) +{ + xml_mktag(xmlTextWriterEndElement(writer), -1); + return 0; +} + +static int _xml_open_incremental_dirs(xmlTextWriterPtr writer, struct incj_path_helper *pm, int offset) +{ + int ret = 0, i = 0; + struct incj_path_element *cur = NULL; + + cur = pm->head; + for (i = 0; i < offset; i++) { + cur = cur->next; + } + + while (cur) { + ret = _xml_open_incremental_dir(writer, cur->d); + if (ret < 0) + break; + cur = cur->next; + } + + return ret; +} + +static int _xml_close_incremental_dirs(xmlTextWriterPtr writer, int pops) +{ + int ret = 0, i = 0; + + for (i = 0; i < pops; i++) { + ret = _xml_close_incremental_dir(writer); + if (ret < 0) + break; + } + + return ret; +} + +static int _xml_write_incremental_delete(xmlTextWriterPtr writer, enum journal_reason reason, struct ltfs_name *name) +{ + /* Open directory tag or file tag based on the provided reason */ + switch (reason) { + case DELETE_DIRECTORY: + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "directory"), -1); + break; + case DELETE_FILE: + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "file"), -1); + break; + default: + ltfsmsg(LTFS_ERR, 17304E, reason); + return -1; + break; + } + + /* Create a name tag */ + xml_mktag(_xml_write_nametype(writer, "name", name), -1); + + /* Create a empty deleted tag */ + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "deleted"), -1); + xml_mktag(xmlTextWriterEndElement(writer), -1); + + /* Close directory or file tag */ + xml_mktag(xmlTextWriterEndElement(writer), -1); + + return 0; +} + +static int _xml_goto_increment_parent(xmlTextWriterPtr writer, + struct jentry *ent, struct incj_path_helper **cur, + struct ltfs_volume *vol) +{ + int ret = 0, matches = 0, pops = 0; + bool perfect_match = false; + char *parent_path = NULL, *filename = NULL; + struct incj_path_helper *new = NULL; + + parent_path = strdup(ent->id.full_path); + if (!parent_path) { + ltfsmsg(LTFS_ERR, 10001E, "parent path for traveling incremental index dirs"); + return -LTFS_NO_MEMORY; + } + + fs_split_path(parent_path, &filename, strlen(parent_path) + 1); + if (strlen(parent_path) == 0) { + parent_path = "/"; + } + + ret = incj_create_path_helper(parent_path, &new, vol); + if (ret < 0) { + free(parent_path); + return ret; + } + + ret = incj_compare_path(*cur, new, &matches, &pops, &perfect_match); + if (ret < 0) { + incj_destroy_path_helper(new); + free(parent_path); + return ret; + } + + if (pops) { + ret = _xml_close_incremental_dirs(writer, pops); + if (ret < 0) { + incj_destroy_path_helper(new); + free(parent_path); + return ret; + } + } + + if (!perfect_match) + ret = _xml_open_incremental_dirs(writer, new, matches); + + if (!ret) { + incj_destroy_path_helper(*cur); + *cur = new; + } else { + incj_destroy_path_helper(new); + } + + if (strcmp(parent_path, "/")) + free(parent_path); + + return ret; +} + +/** + * Write directory tags for incremental index based on current incremental journal information + * @param writer output pointer + * @param vol ltfs volume + * @param offset_c file pointer to write offest cache + * @param sync_list file pointer to write sync file list + * @return 0 on success or negative on failure + */ +static int _xml_write_inc_journal(xmlTextWriterPtr writer, struct ltfs_volume *vol, + struct ltfsee_cache* offset_c, struct ltfsee_cache* sync_list) +{ + int ret = 0; + bool failed = false; + struct ltfsee_cache offset = {NULL, 0}; /* Cache structure for file offset cache */ + struct ltfsee_cache list = {NULL, 0}; /* Cache structure for sync list */ + struct jentry *ent = NULL, *tmp = NULL; + struct incj_path_helper *cur_parent = NULL; + + /* Create a directory tag for root */ + ret = _xml_open_incrental_root(writer, vol); + if (ret < 0) { + return ret; + } + + ret = incj_create_path_helper("/", &cur_parent, vol); + if (ret < 0) { + return ret; + } + + /* Crawl incremental journal and generate XML tags */ + incj_sort(vol); + HASH_ITER(hh, vol->journal, ent, tmp) { + if (!failed) { + ret = _xml_goto_increment_parent(writer, ent, &cur_parent, vol); + if (!ret) { + switch (ent->reason) { + case CREATE: + /* TODO: Need to support sync cache and offset cache */ + if (ent->dentry->isdir) { + /* Create XML recursively */ + ret = _xml_write_dirtree(writer, ent->dentry, vol->index, &offset, &list); + } else { + /* Create XML for a file */ + ret = _xml_write_file(writer, ent->dentry, &offset, &list); + } + break; + case MODIFY: + if (ent->dentry->isdir) { + /* Create XML for dir (no recursive) */ + ret = _xml_write_incremental_dir(writer, ent->dentry); + } else { + /* Create XML for a file */ + ret = _xml_write_file(writer, ent->dentry, &offset, &list); + } + break; + case DELETE_FILE: + case DELETE_DIRECTORY: + /* Create a delete tag */ + ret = _xml_write_incremental_delete(writer, ent->reason, &ent->name); + break; + default: + ltfsmsg(LTFS_ERR, 17303E, ent->reason); + ret = -LTFS_UNEXPECTED_VALUE; + break; + } + + if (ret < 0) + failed = true; + } else + failed = true; + } + HASH_DEL(vol->journal, ent); + incj_dispose_jentry(ent); + } + + if (cur_parent) { + incj_destroy_path_helper(cur_parent); + } + + /* Clear created directory just in case */ + incj_clear(vol); + + /* Close the directory tag for root */ + ret = _xml_close_incrental_root(writer); + + return ret; +} + +/** + * Generate an XML schema, sending it to a user-provided output (memory or file). + * Note: this function does very little input validation; any user-provided information + * must be verified by the caller. + * @param writer the XML writer to send output to + * @param priv LTFS data + * @param pos position on tape where the schema will be written + * @return 0 on success, negative on failure + */ +static int _xml_write_incremental_schema(xmlTextWriterPtr writer, const char *creator, struct ltfs_volume *vol) +{ + int ret; + size_t i; + char *update_time; + struct ltfsee_cache offset = {NULL, 0}; /* Cache structure for file offset cache */ + struct ltfsee_cache list = {NULL, 0}; /* Cache structure for sync list */ + struct ltfs_index *idx = vol->index; + + ret = xml_format_time(idx->mod_time, &update_time); + if (!update_time) + return -1; + else if (ret == LTFS_TIME_OUT_OF_RANGE) + ltfsmsg(LTFS_WARN, 17224W, "modifytime", (unsigned long long)idx->mod_time.tv_sec); + + ret = xmlTextWriterStartDocument(writer, NULL, "UTF-8", NULL); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17057E, ret); + return -1; + } + + xmlTextWriterSetIndent(writer, 1); + /* Define INDENT_INDEXES to write Indexes to tape with full indentation. + * This is normally a waste of space, but it may be useful for debugging. */ +#ifdef INDENT_INDEXES + xmlTextWriterSetIndentString(writer, BAD_CAST " "); +#else + xmlTextWriterSetIndentString(writer, BAD_CAST ""); +#endif + + /* write index properties */ + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "ltfsincrementalindex"), -1); + xml_mktag(xmlTextWriterWriteAttribute(writer, BAD_CAST "version", + BAD_CAST LTFS_INDEX_VERSION_STR), -1); + xml_mktag(xmlTextWriterWriteElement(writer, BAD_CAST "creator", BAD_CAST creator), -1); + if (idx->commit_message && strlen(idx->commit_message)) { + xml_mktag(xmlTextWriterWriteFormatElement(writer, BAD_CAST "comment", + "%s", BAD_CAST (idx->commit_message)), -1); + } + xml_mktag(xmlTextWriterWriteElement(writer, BAD_CAST "volumeuuid", BAD_CAST idx->vol_uuid), -1); + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST "generationnumber", "%u", idx->generation), -1); + xml_mktag(xmlTextWriterWriteElement(writer, BAD_CAST "updatetime", BAD_CAST update_time), -1); + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "location"), -1); + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST "partition", "%c", idx->selfptr_inc.partition), -1); + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST "startblock", "%"PRIu64, idx->selfptr_inc.block), -1); + xml_mktag(xmlTextWriterEndElement(writer), -1); + if (idx->backptr.block) { + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "previousgenerationlocation"), -1); + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST "partition", "%c", idx->backptr.partition), -1); + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST "startblock", "%"PRIu64, idx->backptr.block), -1); + xml_mktag(xmlTextWriterEndElement(writer), -1); + } + + if (idx->backptr_inc.block) { + xml_mktag(xmlTextWriterStartElement(writer, BAD_CAST "previousincrementallocation"), -1); + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST "partition", "%c", idx->backptr_inc.partition), -1); + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST "startblock", "%"PRIu64, idx->backptr_inc.block), -1); + xml_mktag(xmlTextWriterEndElement(writer), -1); + } + + xml_mktag(xmlTextWriterWriteFormatElement( + writer, BAD_CAST NEXTUID_TAGNAME, "%"PRIu64, idx->uid_number), -1); + + { + char *value = NULL; + + switch (idx->vollock) { + case LOCKED_MAM: + asprintf(&value, "locked"); + break; + case PERMLOCKED_MAM: + asprintf(&value, "permlocked"); + break; + default: + asprintf(&value, "unlocked"); + break; + } + + if (value) + xml_mktag(xmlTextWriterWriteElement(writer, BAD_CAST "volumelockstate", BAD_CAST value), -1); + + free(value); + } + + /* Create XML of update */ + xml_mktag(_xml_write_inc_journal(writer, vol, &offset, &list), -1); + + /* Save unrecognized tags */ + if (idx->tag_count > 0) { + for (i=0; itag_count; ++i) { + if (xmlTextWriterWriteRaw(writer, idx->preserved_tags[i]) < 0) { + ltfsmsg(LTFS_ERR, 17092E, __FUNCTION__); + return -1; + } + } + } + + xml_mktag(xmlTextWriterEndElement(writer), -1); + ret = xmlTextWriterEndDocument(writer); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 17058E, ret); + return -1; + } + + free(update_time); + return 0; +} +#endif + /************************************************************************************** * Global Functions **************************************************************************************/ @@ -726,7 +1150,7 @@ static int _commit_offset_caches(const char* path, const struct ltfs_index *idx) unlink(offset_name); rename(offset_new, offset_name); fd = open(offset_name, O_RDWR | O_BINARY, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP ); if (fd >= 0) { fsync(fd); close(fd); @@ -747,7 +1171,7 @@ static int _commit_offset_caches(const char* path, const struct ltfs_index *idx) unlink(sync_name); rename(sync_new, sync_name); fd = open(sync_name, O_RDWR | O_BINARY, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP ); if (fd >= 0) { fsync(fd); close(fd); @@ -814,10 +1238,12 @@ int xml_schema_to_file(const char *filename, const char *creator, /** * Generate an XML Index based on the vol->index->root directory tree. * The generated data are written directly to the tape with the appropriate blocksize. + * @param reason the reason of writing an index on tape + * @param type index type to write (shall be LTFS_FULL_INDEX or LTFS_INCREMENTAL_INDEX) * @param vol LTFS volume. * @return 0 on success or a negative value on error. */ -int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) +int xml_schema_to_tape(char *reason, int type, struct ltfs_volume *vol) { int ret, bk = -1; xmlOutputBufferPtr write_buf; @@ -844,8 +1270,8 @@ int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) out_ctx->fd = -1; out_ctx->errno_fd = 0; - if (vol->index_cache_path) - xml_acquire_file_lock(vol->index_cache_path, &out_ctx->fd, &bk, true); + if (vol->index_cache_path_w) + xml_acquire_file_lock(vol->index_cache_path_w, &out_ctx->fd, &bk, true); out_ctx->buf_size = vol->label->blocksize; out_ctx->buf_used = 0; @@ -859,7 +1285,7 @@ int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) if (! write_buf) { ltfsmsg(LTFS_ERR, 17053E); if (out_ctx->fd >= 0) - xml_release_file_lock(vol->index_cache_path, out_ctx->fd, bk, false); + xml_release_file_lock(vol->index_cache_path_w, out_ctx->fd, bk, false); free(out_ctx->buf); free(out_ctx); return -LTFS_LIBXML2_FAILURE; @@ -870,7 +1296,7 @@ int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) if (! writer) { ltfsmsg(LTFS_ERR, 17054E); if (out_ctx->fd >= 0) - xml_release_file_lock(vol->index_cache_path, out_ctx->fd, bk, false); + xml_release_file_lock(vol->index_cache_path_w, out_ctx->fd, bk, false); xmlOutputBufferClose(write_buf); free(out_ctx->buf); free(out_ctx); @@ -880,7 +1306,19 @@ int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) /* Generate the Index. */ asprintf(&creator, "%s - %s", vol->creator, reason); if (creator) { - ret = _xml_write_schema(writer, creator, vol->index); + switch (type) { + case LTFS_FULL_INDEX: + ret = _xml_write_schema(writer, creator, vol->index); + break; +#ifdef FORMAT_SPEC25 + case LTFS_INCREMENTAL_INDEX: + ret = _xml_write_incremental_schema(writer, creator, vol); + break; +#endif + default: + ret = -LTFS_BAD_INDEX_TYPE; + break; + } if (ret < 0) { ltfsmsg(LTFS_ERR, 17055E, ret); } @@ -893,7 +1331,7 @@ int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) else if (out_ctx->errno_fd) ret = out_ctx->errno_fd; if (out_ctx->fd >= 0) - xml_release_file_lock(vol->index_cache_path, out_ctx->fd, bk, true); + xml_release_file_lock(vol->index_cache_path_w, out_ctx->fd, bk, true); } else { /* New index is successfully sent to the internal buffer of tape drive */ immed = (strcmp(reason, SYNC_FORMAT) == 0); /* Use immediate write FM only at format */ @@ -903,14 +1341,14 @@ int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) * All buffered data, new index and following FM is written on tape correctly. * It's time to unveil the offset cache and sync cache to other programs. */ - if (vol->index_cache_path) - _commit_offset_caches(vol->index_cache_path, vol->index); + if (vol->index_cache_path_w) + _commit_offset_caches(vol->index_cache_path_w, vol->index); } else { ltfsmsg(LTFS_ERR, 11084E, ret); } if (out_ctx->fd >= 0) - xml_release_file_lock(vol->index_cache_path, out_ctx->fd, bk, false); + xml_release_file_lock(vol->index_cache_path_w, out_ctx->fd, bk, false); } /* Update the creator string */ @@ -928,7 +1366,7 @@ int xml_schema_to_tape(char *reason, struct ltfs_volume *vol) } else { ltfsmsg(LTFS_ERR, 10001E, "xml_schema_to_tape: creator string"); xmlFreeTextWriter(writer); - xml_release_file_lock(vol->index_cache_path, out_ctx->fd, bk, true); + xml_release_file_lock(vol->index_cache_path_w, out_ctx->fd, bk, true); ret = -LTFS_NO_MEMORY; } diff --git a/src/ltfs_copyright.h b/src/ltfs_copyright.h index ff128c89..97769fb2 100644 --- a/src/ltfs_copyright.h +++ b/src/ltfs_copyright.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -44,6 +44,10 @@ ** IBM Almaden Research Center ** bbiskebo@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -54,12 +58,16 @@ extern "C" { #endif -#define LTFS_COPYRIGHT_0 "Licensed Materials - Property of IBM" -#define LTFS_COPYRIGHT_1 "IBM Spectrum Archive Single Drive Edition" -#define LTFS_COPYRIGHT_2 "(C) Copyright IBM Corp. 2011, 2017 All Rights Reserved" -#define LTFS_COPYRIGHT_3 "US Government Users Restricted Rights - Use, duplication or " -#define LTFS_COPYRIGHT_4 "disclosure restricted by GSA ADP Schedule Contract with" -#define LTFS_COPYRIGHT_5 "IBM Corp." +#define LTFS_COPYRIGHT_0 "Copyright 2010, 2022 The LTFS project. All rights reserved." +#define LTFS_COPYRIGHT_1 "Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:" +#define LTFS_COPYRIGHT_2 "* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer." +#define LTFS_COPYRIGHT_3 "* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution." +#define LTFS_COPYRIGHT_4 "* Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. " +#define LTFS_COPYRIGHT_5 "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED " \ + "WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES " \ + "(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND " \ + "ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS " \ + "SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." #ifdef __cplusplus } diff --git a/src/ltfs_fuse.c b/src/ltfs_fuse.c index f26501d2..4e12188f 100644 --- a/src/ltfs_fuse.c +++ b/src/ltfs_fuse.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -54,6 +54,9 @@ ** ************************************************************************************* */ + +#include /* for ULONG_MAX */ + #include "ltfs_fuse.h" #include "libltfs/ltfs_fsops.h" #include "libltfs/iosched.h" @@ -68,7 +71,7 @@ #include "libltfs/arch/win/win_util.h" #endif -#if (__WORDSIZE == 64) +#if (__WORDSIZE == 64 || ULONG_MAX == 0xffffffffffffffffUL) #define FILEHANDLE_TO_STRUCT(fh) ((struct ltfs_file_handle *)(uint64_t)(fh)) #define STRUCT_TO_FILEHANDLE(de) ((uint64_t)(de)) #else @@ -329,17 +332,21 @@ int ltfs_fuse_statfs(const char *path, struct statvfs *buf) stats->f_bfree = blockstat.remaining_dp; /* Remaining tape capacity */ stats->f_bavail = stats->f_bfree; /* Blocks available for normal user (ignored) */ - stats->f_files = UINT64_MAX; - stats->f_ffree = UINT64_MAX - ltfs_get_file_count(priv->data); +#ifdef __APPLE__ + stats->f_files = UINT_MAX; + stats->f_ffree = UINT_MAX - ltfs_get_file_count(priv->data); memcpy(buf, stats, sizeof(struct statvfs)); -#ifdef __APPLE__ /* With MacFUSE, we use an f_frsize not equal to the file system block size. * Need to adjust the block counts so they're in units of the reported f_frsize. */ double scale = ltfs_get_blocksize(priv->data) / (double)stats->f_frsize; buf->f_blocks *= scale; buf->f_bfree *= scale; buf->f_bavail *= scale; +#else + stats->f_files = UINT64_MAX; + stats->f_ffree = UINT64_MAX - ltfs_get_file_count(priv->data); + memcpy(buf, stats, sizeof(struct statvfs)); #endif /* __APPLE__ */ ltfs_request_trace(FUSE_REQ_EXIT(REQ_STATFS), 0, 0); @@ -443,8 +450,10 @@ int ltfs_fuse_release(const char *path, struct fuse_file_info *fi) open_write = (((fi->flags & O_WRONLY) == O_WRONLY) || ((fi->flags & O_RDWR) == O_RDWR)); ret = ltfs_fsops_close(file->file_info->dentry_handle, dirty, open_write, true, priv->data); - if (write_index) - ltfs_sync_index(SYNC_CLOSE, true, priv->data); + if (write_index) { + ltfs_set_commit_message_reason(SYNC_CLOSE, priv->data); + ltfs_sync_index(SYNC_CLOSE, true, LTFS_INDEX_AUTO, priv->data); + } _file_close(file->file_info, priv); _free_ltfs_file_handle(file); @@ -1160,11 +1169,9 @@ void ltfs_fuse_umount(void *userdata) if (kmi_initialized(priv->data)) kmi_destroy(priv->data); + ltfs_set_commit_message_reason(SYNC_UNMOUNT, priv->data); ltfs_unmount(SYNC_UNMOUNT, priv->data); - if (priv->capture_index) - ltfs_save_index_to_disk(priv->work_directory, SYNC_UNMOUNT, false, priv->data); - ltfs_request_trace(FUSE_REQ_EXIT(REQ_UNMOUNT), 0, 0); } diff --git a/src/ltfs_fuse.h b/src/ltfs_fuse.h index 07a6cd35..d6e97b2c 100644 --- a/src/ltfs_fuse.h +++ b/src/ltfs_fuse.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -48,6 +48,10 @@ ** IBM Almaden Research Center ** lucasvr@us.ibm.com ** +** Atsushi Abe +** IBM Tokyo Lab., Japan +** piste@jp.ibm.com +** ************************************************************************************* */ @@ -127,7 +131,7 @@ struct ltfs_fuse_data { unsigned int rollback_gen; /**< Target generation to roll back mount */ int release_device; /**< Release device? */ int allow_other; /**< Allow all users to access the volume? */ - int capture_index; /**< Capture index information to work directory at unmount */ + char *capture_dir; /**< Directory to capture index information */ char *symlink_str; /**< Symbolic Link type fetched by option (live or posix)*/ char *str_append_only_mode; /**< option sting of scsi_append_only_mode */ int append_only_mode; /**< Use append-only mode */ diff --git a/src/main.c b/src/main.c index 6b8298be..0add6121 100644 --- a/src/main.c +++ b/src/main.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -133,7 +133,7 @@ static struct fuse_opt ltfs_options[] = { LTFS_OPT("release_device", release_device, 1), LTFS_OPT("allow_other", allow_other, 1), LTFS_OPT("noallow_other", allow_other, 0), - LTFS_OPT("capture_index", capture_index, 1), + LTFS_OPT("capture_index=%s", capture_dir, 0), LTFS_OPT("symlink_type=%s", symlink_str, 0), LTFS_OPT("scsi_append_only_mode=%s", str_append_only_mode, 0), LTFS_OPT_KEY("-a", KEY_ADVANCED_HELP), @@ -146,35 +146,39 @@ static struct fuse_opt ltfs_options[] = { void single_drive_advanced_usage(const char *default_driver, struct ltfs_fuse_data *priv) { - ltfsresult(14401I); /* LTFS options: */ - ltfsresult(14413I, LTFS_CONFIG_FILE); /* -o config_file= */ - ltfsresult(14404I, LTFS_DEFAULT_WORK_DIR); /* -o work_directory= */ - ltfsresult(14414I); /* -o atime */ - ltfsresult(14440I); /* -o noatime */ - ltfsresult(14415I, default_driver); /* -o tape_backend= */ + ltfsresult(14401I); /* LTFS options: */ + ltfsresult(14403I); /* -o devname= */ + ltfsresult(14413I, LTFS_CONFIG_FILE); /* -o config_file= */ + ltfsresult(14404I, LTFS_DEFAULT_WORK_DIR); /* -o work_directory= */ + ltfsresult(14414I); /* -o atime */ + ltfsresult(14440I); /* -o noatime */ + ltfsresult(14415I, default_driver); /* -o tape_backend= */ ltfsresult(14416I, config_file_get_default_plugin("iosched", priv->config)); /* -o iosched_backend= */ - ltfsresult(14455I, config_file_get_default_plugin("kmi", priv->config)); /* -o kmi_backend= */ - ltfsresult(14417I); /* -o umask= */ - ltfsresult(14418I); /* -o fmask= */ - ltfsresult(14419I); /* -o dmask= */ + ltfsresult(14455I, config_file_get_default_plugin("kmi", priv->config)); /* -o kmi_backend= */ + ltfsresult(14417I); /* -o umask= */ + ltfsresult(14418I); /* -o fmask= */ + ltfsresult(14419I); /* -o dmask= */ ltfsresult(14420I, LTFS_MIN_CACHE_SIZE_DEFAULT); /* -o min_pool_size= */ ltfsresult(14421I, LTFS_MAX_CACHE_SIZE_DEFAULT); /* -o max_pool_size= */ - ltfsresult(14422I); /* -o rules= */ - ltfsresult(14423I); /* -o quiet */ - ltfsresult(14405I); /* -o trace */ - ltfsresult(14467I); /* -o syslogtrace */ - ltfsresult(14424I); /* -o fulltrace */ - ltfsresult(14441I, LTFS_INFO); /* -o verbose= */ - ltfsresult(14425I); /* -o eject */ - ltfsresult(14439I); /* -o noeject */ - ltfsresult(14427I, LONG_MAX / 60); /* -o sync_type=type */ - ltfsresult(14443I); /* -o force_mount_no_eod */ - ltfsresult(14436I); /* -o device_list */ - ltfsresult(14437I); /* -o rollback_mount */ - ltfsresult(14448I); /* -o release_device */ - ltfsresult(14456I); /* -o capture_index */ - ltfsresult(14463I); /* -o scsi_append_only_mode= */ - ltfsresult(14406I); /* -a */ + ltfsresult(14422I); /* -o rules= */ + ltfsresult(14423I); /* -o quiet */ + ltfsresult(14405I); /* -o trace */ + ltfsresult(14467I); /* -o syslogtrace */ + ltfsresult(14424I); /* -o fulltrace */ + ltfsresult(14441I, LTFS_INFO); /* -o verbose= */ + ltfsresult(14425I); /* -o eject */ + ltfsresult(14439I); /* -o noeject */ + ltfsresult(14427I, LONG_MAX / 60); /* -o sync_type=type */ + ltfsresult(14443I); /* -o force_mount_no_eod */ + ltfsresult(14436I); /* -o device_list */ + ltfsresult(14437I); /* -o rollback_mount */ + ltfsresult(14448I); /* -o release_device */ + ltfsresult(14461I); /* -o symlink_type=type */ + ltfsresult(14456I); /* -o capture_index */ + ltfsresult(14463I); /* -o scsi_append_only_mode= */ + ltfsresult(14406I); /* -a */ + ltfsresult(14407I); /* -V, --version */ + ltfsresult(14408I); /* -h, --help */ /* TODO: future use for WORM */ /* set worm rollback flag and rollback_str by this option */ /* ltfsresult(14468I); */ /* -o rollback_mount_no_eod */ @@ -194,30 +198,30 @@ void usage(char *progname, struct ltfs_fuse_data *priv) if (ret == 0) default_device = ltfs_default_device_name(priv->tape_plugin.ops); - ltfsresult(14400I, progname); /* usage: %s mountpoint [options] */ + ltfsresult(14400I, progname); /* usage: %s mountpoint [options] */ fprintf(stderr, "\n"); - ltfsresult(14401I); /* LTFS options: */ + ltfsresult(14401I); /* LTFS options: */ if (default_device) - ltfsresult(14402I, default_device); /* -o devname= */ + ltfsresult(14402I, default_device); /* -o devname= */ else - ltfsresult(14403I); /* -o devname= */ - ltfsresult(14404I, LTFS_DEFAULT_WORK_DIR); /* -o work_directory= */ - ltfsresult(14405I); /* -o trace */ - ltfsresult(14425I); /* -o eject */ - ltfsresult(14427I, LONG_MAX / 60); /* -o sync_type=type */ - ltfsresult(14443I); /* -o force_mount_no_eod */ - ltfsresult(14436I); /* -o device_list */ - ltfsresult(14437I); /* -o rollback_mount */ - ltfsresult(14448I); /* -o release_device */ - ltfsresult(14461I); /* -o symlink_type=type */ - ltfsresult(14406I); /* -a */ - ltfsresult(14407I); /* -V, --version */ - ltfsresult(14408I); /* -h, --help */ + ltfsresult(14403I); /* -o devname= */ + ltfsresult(14404I, LTFS_DEFAULT_WORK_DIR); /* -o work_directory= */ + ltfsresult(14405I); /* -o trace */ + ltfsresult(14425I); /* -o eject */ + ltfsresult(14427I, LONG_MAX / 60); /* -o sync_type=type */ + ltfsresult(14443I); /* -o force_mount_no_eod */ + ltfsresult(14436I); /* -o device_list */ + ltfsresult(14437I); /* -o rollback_mount */ + ltfsresult(14448I); /* -o release_device */ + ltfsresult(14461I); /* -o symlink_type=type */ + ltfsresult(14406I); /* -a */ + ltfsresult(14407I); /* -V, --version */ + ltfsresult(14408I); /* -h, --help */ fprintf(stderr, "\n"); - ltfsresult(14409I); /* FUSE options: */ - ltfsresult(14410I); /* -o umask=M */ - ltfsresult(14411I); /* -o uid=N */ - ltfsresult(14412I); /* -o gid=N */ + ltfsresult(14409I); /* FUSE options: */ + ltfsresult(14410I); /* -o umask=M */ + ltfsresult(14411I); /* -o uid=N */ + ltfsresult(14412I); /* -o gid=N */ fprintf(stderr, "\n"); fprintf(stderr, "\n"); @@ -704,7 +708,7 @@ int main(int argc, char **argv) if (priv->device_list) { ret = show_device_list(priv); ltfs_finish(); - return (ret != 0) ? 0 : 1; + return ret ? 1 : 0; } /* Validate sync option */ @@ -877,11 +881,14 @@ int main(int argc, char **argv) /* Make sure we have a device name */ if (! priv->devname) { - priv->devname = ltfs_default_device_name(priv->tape_plugin.ops); - if (! priv->devname) { - /* The backend \'%s\' does not have a default device */ - ltfsmsg(LTFS_ERR, 14009E, priv->tape_backend_name); - return 1; + /* Accept no devname when accessible index file is specified by '-o rollback_mount' */ + if ( !priv->rollback_str || access(priv->rollback_str, R_OK) ) { + priv->devname = ltfs_default_device_name(priv->tape_plugin.ops); + if (! priv->devname) { + /* The backend \'%s\' does not have a default device */ + ltfsmsg(LTFS_ERR, 14009E, priv->tape_backend_name); + return 1; + } } } @@ -922,9 +929,10 @@ int main(int argc, char **argv) int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) { - int ret, altret; + int ret, altret, fd = -1; char *index_rules_utf8; - char fsname[strlen(priv->devname) + 16]; + char *fsname_base = "-ofsname=ltfs:"; + char *fsname; char *invalid_start; #ifdef __APPLE__ char *opt_volname = NULL; @@ -934,6 +942,18 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) int i; bool is_worm = false, is_ro = false; + if (priv->devname) { + fsname = calloc(1, strlen(fsname_base) + strlen(priv->devname) + 1); + } else if (priv->rollback_str) { + fsname = calloc(1, strlen(fsname_base) + strlen(priv->rollback_str) + 1); + } else { + fsname = calloc(1, strlen(fsname_base) + 1); + } + if (!fsname) { + /* Memory allocation failed */ + ltfsmsg(LTFS_ERR, 10001E, "fsname"); + return -ENOMEM; + } /* Setup signal handler to terminate cleanly */ ret = ltfs_set_signal_handlers(); if (ret < 0) { @@ -943,11 +963,13 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) /* Validate rollback_mount option */ if (priv->rollback_str) { - errno = 0; - priv->rollback_gen = strtoul(priv->rollback_str, &invalid_start, 0); - if( (*invalid_start != '\0') || priv->rollback_gen == 0 ) { - ltfsmsg(LTFS_ERR, 14091E, priv->rollback_str); - return 1; + if (access(priv->rollback_str, R_OK)) { + errno = 0; + priv->rollback_gen = strtoul(priv->rollback_str, &invalid_start, 0); + if( (*invalid_start != '\0') || priv->rollback_gen == 0 ) { + ltfsmsg(LTFS_ERR, 14091E, priv->rollback_str); + return 1; + } } } @@ -983,7 +1005,12 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) } /* Set file system name to "ltfs:devname" in case FUSE doesn't pick it up */ - snprintf(fsname, sizeof(fsname), "-ofsname=ltfs:%s", priv->devname); + strcpy(fsname, fsname_base); + if (priv->devname) { + strcat(fsname, priv->devname); + } else if (priv->rollback_str) { + strcat(fsname, priv->rollback_str); + } ret = fuse_opt_add_arg(args, fsname); if (ret < 0) { /* Could not enable FUSE option */ @@ -1000,145 +1027,163 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) ltfs_use_atime(priv->atime, priv->data); ltfs_set_work_dir(priv->work_directory, priv->data); - if (ltfs_device_open(priv->devname, priv->tape_plugin.ops, priv->data) < 0) { - /* Could not open device */ - ltfsmsg(LTFS_ERR, 10004E, priv->devname); - ltfs_volume_free(&priv->data); - return 1; - } - - if (priv->release_device) { - ltfs_release_medium(priv->data); - ltfs_device_close(priv->data); - ltfs_volume_free(&priv->data); - return 0; - } - - /* Parse backend options */ - if (ltfs_parse_tape_backend_opts(args, priv->data)) { - /* Backend option parsing failed */ - ltfsmsg(LTFS_ERR, 14012E); - ltfs_volume_free(&priv->data); - return 1; - } - - if (priv->kmi_backend_name) { - if (kmi_init(&priv->kmi_plugin, priv->data) < 0) { - /* Encryption function disabled. */ - ltfsmsg(LTFS_ERR, 14089E); + if (priv->devname) { + if (ltfs_device_open(priv->devname, priv->tape_plugin.ops, priv->data) < 0) { + /* Could not open device */ + ltfsmsg(LTFS_ERR, 10004E, priv->devname); ltfs_volume_free(&priv->data); return 1; } - if (ltfs_parse_kmi_backend_opts(args, priv->data)) { + if (priv->release_device) { + ltfs_release_medium(priv->data); + ltfs_device_close(priv->data); + ltfs_volume_free(&priv->data); + return 0; + } + + /* Parse backend options */ + if (ltfs_parse_tape_backend_opts(args, priv->data)) { /* Backend option parsing failed */ - ltfsmsg(LTFS_ERR, 14090E); + ltfsmsg(LTFS_ERR, 14012E); ltfs_volume_free(&priv->data); return 1; } - if (tape_clear_key(priv->data->device, priv->data->kmi_handle) < 0) - return 1; - } + if (priv->kmi_backend_name) { + if (kmi_init(&priv->kmi_plugin, priv->data) < 0) { + /* Encryption function disabled. */ + ltfsmsg(LTFS_ERR, 14089E); + ltfs_volume_free(&priv->data); + return 1; + } - /* Setup tape drive */ - ltfs_load_tape(priv->data); - ret = ltfs_wait_device_ready(priv->data); - if (ret < 0) { - ltfsmsg(LTFS_ERR, 14075E); - ltfs_volume_free(&priv->data); - return 1; - } + if (ltfs_parse_kmi_backend_opts(args, priv->data)) { + /* Backend option parsing failed */ + ltfsmsg(LTFS_ERR, 14090E); + ltfs_volume_free(&priv->data); + return 1; + } - priv->data->append_only_mode = (bool)priv->append_only_mode; - if (ltfs_setup_device(priv->data)) { - ltfsmsg(LTFS_ERR, 14075E); - ltfs_volume_free(&priv->data); - return 1; - } + if (tape_clear_key(priv->data->device, priv->data->kmi_handle) < 0) + return 1; + } - /* Check EOD validation is skipped or not */ - if (priv->skip_eod_check) { - ltfsmsg(LTFS_INFO, 14076I); - ltfsmsg(LTFS_INFO, 14077I); - ltfs_set_eod_check(! priv->skip_eod_check, priv->data); - } + /* Setup tape drive */ + ltfs_load_tape(priv->data); + ret = ltfs_wait_device_ready(priv->data); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 14075E); + ltfs_volume_free(&priv->data); + return 1; + } - /* Validate symbolic link type */ - priv->data->livelink = false; - if (priv->symlink_str) { - if (strcasecmp(priv->symlink_str, "live") == 0) - priv->data->livelink = true; - else if (strcasecmp(priv->symlink_str, "posix") == 0) - priv->data->livelink = false; - else { - ltfsmsg(LTFS_ERR, 14093E, priv->symlink_str); + priv->data->append_only_mode = (bool)priv->append_only_mode; + if (ltfs_setup_device(priv->data)) { + ltfsmsg(LTFS_ERR, 14075E); + ltfs_volume_free(&priv->data); return 1; } - ltfsmsg(LTFS_INFO, 14092I, priv->symlink_str); - } - /* Mount the volume */ - ltfs_set_traverse_mode(TRAVERSE_BACKWARD, priv->data); - if (ltfs_mount(false, false, false, false, priv->rollback_gen, priv->data) < 0) { - ltfsmsg(LTFS_ERR, 14013E); - ltfs_volume_free(&priv->data); - return 1; - } + /* Check EOD validation is skipped or not */ + if (priv->skip_eod_check) { + ltfsmsg(LTFS_INFO, 14076I); + ltfsmsg(LTFS_INFO, 14077I); + ltfs_set_eod_check(! priv->skip_eod_check, priv->data); + } - ret = tape_get_worm_status(priv->data->device, &is_worm); - if (ret != 0 || is_worm) { - ltfsmsg(LTFS_ERR, 14116E, ret); - ltfs_volume_free(&priv->data); - return 1; - } + /* Validate symbolic link type */ + priv->data->livelink = false; + if (priv->symlink_str) { + if (strcasecmp(priv->symlink_str, "live") == 0) + priv->data->livelink = true; + else if (strcasecmp(priv->symlink_str, "posix") == 0) + priv->data->livelink = false; + else { + ltfsmsg(LTFS_ERR, 14093E, priv->symlink_str); + return 1; + } + ltfsmsg(LTFS_INFO, 14092I, priv->symlink_str); + } - /* Set up index criteria */ - if (priv->index_rules) { - ret = pathname_format(priv->index_rules, &index_rules_utf8, false, false); - if (ret < 0) { - /* Could not format data placement rules. */ - ltfsmsg(LTFS_ERR, 14016E, ret); - ltfs_volume_free(&priv->data); - return 1; + /* Mount the volume */ + ltfs_set_traverse_mode(TRAVERSE_BACKWARD, priv->data); + if (priv->rollback_str) { + if (ltfs_mount_indexfile(priv->rollback_str, true, priv->data) < 0) { + ltfsmsg(LTFS_ERR, 14013E, "index file"); + ltfs_volume_free(&priv->data); + return 1; + } + } else { + if (ltfs_mount(false, false, false, false, priv->rollback_gen, priv->data) < 0) { + ltfsmsg(LTFS_ERR, 14013E, "device"); + ltfs_volume_free(&priv->data); + return 1; + } } - ret = ltfs_override_policy(index_rules_utf8, false, priv->data); - free(index_rules_utf8); - if (ret == -LTFS_POLICY_IMMUTABLE) { - /* Volume doesn't allow override. Ignoring user-specified criteria. */ - ltfsmsg(LTFS_WARN, 14015W); - } else if (ret < 0) { - /* Could not parse data placement rules */ - ltfsmsg(LTFS_ERR, 14017E, ret); + + ret = tape_get_worm_status(priv->data->device, &is_worm); + if (ret != 0 || is_worm) { + ltfsmsg(LTFS_ERR, 14116E, ret); ltfs_volume_free(&priv->data); return 1; } - } - /* Configure I/O scheduler cache */ - ltfs_set_scheduler_cache(priv->min_pool_size, priv->max_pool_size, priv->data); - - /* mount read-only if underlying medium is write-protected */ - ret = ltfs_get_tape_readonly(priv->data); - switch (ret) { - case 0: - case -LTFS_WRITE_PROTECT: - case -LTFS_WRITE_ERROR: - case -LTFS_NO_SPACE: - case -LTFS_LESS_SPACE: - case -LTFS_RDONLY_DEN_DRV: - /* Do nothing */ - break; - default: - /* Fail immidiatly when return code is NOT success or NOT possible R/O related errors */ - /* Could not get read-only status of medium */ - ltfsmsg(LTFS_ERR, 14018E); + /* Set up index criteria */ + if (priv->index_rules) { + ret = pathname_format(priv->index_rules, &index_rules_utf8, false, false); + if (ret < 0) { + /* Could not format data placement rules. */ + ltfsmsg(LTFS_ERR, 14016E, ret); + ltfs_volume_free(&priv->data); + return 1; + } + ret = ltfs_override_policy(index_rules_utf8, false, priv->data); + free(index_rules_utf8); + if (ret == -LTFS_POLICY_IMMUTABLE) { + /* Volume doesn't allow override. Ignoring user-specified criteria. */ + ltfsmsg(LTFS_WARN, 14015W); + } else if (ret < 0) { + /* Could not parse data placement rules */ + ltfsmsg(LTFS_ERR, 14017E, ret); + ltfs_volume_free(&priv->data); + return 1; + } + } + + /* Configure I/O scheduler cache */ + ltfs_set_scheduler_cache(priv->min_pool_size, priv->max_pool_size, priv->data); + + /* mount read-only if underlying medium is write-protected */ + ret = ltfs_get_tape_readonly(priv->data); + switch (ret) { + case 0: + case -LTFS_WRITE_PROTECT: + case -LTFS_WRITE_ERROR: + case -LTFS_NO_SPACE: + case -LTFS_LESS_SPACE: + case -LTFS_RDONLY_DEN_DRV: + /* Do nothing */ + break; + default: + /* Fail immidiatly when return code is NOT success or NOT possible R/O related errors */ + /* Could not get read-only status of medium */ + ltfsmsg(LTFS_ERR, 14018E); + ltfs_volume_free(&priv->data); + return 1; + break; + } + } else { + /* try to mount from index file (meta-only mount) */ + if (ltfs_mount_indexfile(priv->rollback_str, false, priv->data) < 0) { + ltfsmsg(LTFS_ERR, 14013E, "index file"); ltfs_volume_free(&priv->data); return 1; - break; + } + ret = 0; } - if (ret < 0 || priv->rollback_gen != 0) { + if (ret < 0 || priv->rollback_gen || priv->rollback_str) { switch (ret) { case -LTFS_WRITE_PROTECT: case -LTFS_WRITE_ERROR: @@ -1171,9 +1216,25 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) is_ro = true; break; default: - /* Rollback mount is specified */ - ltfsmsg(LTFS_INFO, 14072I, priv->rollback_gen); - is_ro = true; + if (!ret && priv->rollback_gen) { + /* Rollback mount is specified */ + ltfsmsg(LTFS_INFO, 14072I, priv->rollback_gen); + is_ro = true; + } else if (!ret && priv->rollback_str) { + if (priv->devname) { + /* Rollback mount (index mount) is specified */ + ltfsmsg(LTFS_INFO, 14119I, priv->rollback_str); + } else { + /* Rollback mount (meta-only mount) is specified */ + ltfsmsg(LTFS_INFO, 14117I, priv->rollback_str); + } + is_ro = true; + } else { + /* Unexpected condition */ + ltfsmsg(LTFS_ERR, 14118E, ret); + ltfs_volume_free(&priv->data); + return 1; + } break; } @@ -1188,10 +1249,42 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) } } + /* Configure index capturing */ + if (priv->capture_dir) { + if (HAVE_BARCODE(priv->data)) + ret = asprintf(&priv->data->index_cache_path_w, "%s/%s.schema", + priv->capture_dir, priv->data->label->barcode); + else + ret = asprintf(&priv->data->index_cache_path_w, "%s/%s.schema", + priv->capture_dir, priv->data->label->vol_uuid); + + if (ret > 0) { + fd = open(priv->data->index_cache_path_w, O_WRONLY | O_BINARY | O_CREAT, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); + if (fd < 0) { + ltfsmsg(LTFS_WARN, 14120W, priv->capture_dir, errno); + free(priv->data->index_cache_path_w); + priv->data->index_cache_path_w = NULL; + } else { + ltfsmsg(LTFS_INFO, 14121I, priv->data->index_cache_path_w); + close(fd); + fd = -1; + } + } else { + ltfsmsg(LTFS_ERR, 10001E, "capture_dir"); + ltfs_volume_free(&priv->data); + return 1; + } + } else { + ltfsmsg(LTFS_INFO, 14122I); + ret = 0; + } + /* Cleanup signal handler */ ret = ltfs_unset_signal_handlers(); if (ret < 0) { ltfsmsg(LTFS_ERR, 10014E); + ltfs_volume_free(&priv->data); return 1; } @@ -1242,6 +1335,9 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) ltfsmsg(LTFS_INFO, 14112I); ltfsmsg(LTFS_INFO, 14113I); ret = fuse_main(args->argc, args->argv, <fs_ops, priv); + if (ret != 0) { + ltfsmsg(LTFS_WARN, 14123W, ret); + } /* Setup signal handler again to terminate cleanly */ ret = ltfs_set_signal_handlers(); @@ -1250,7 +1346,7 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) return 1; } - if (priv->eject) + if (priv->devname && priv->eject) ltfs_eject_tape(false, priv->data); /* close the volume */ @@ -1258,7 +1354,8 @@ int single_drive_main(struct fuse_args *args, struct ltfs_fuse_data *priv) if (opt_volname) free(opt_volname); #endif - ltfs_device_close(priv->data); + if (priv->devname) + ltfs_device_close(priv->data); ltfs_volume_free(&priv->data); ltfs_unset_signal_handlers(); diff --git a/src/tape_drivers/freebsd/cam/.gitignore b/src/tape_drivers/freebsd/cam/.gitignore index b4630e16..2f6e991c 100644 --- a/src/tape_drivers/freebsd/cam/.gitignore +++ b/src/tape_drivers/freebsd/cam/.gitignore @@ -1,3 +1,4 @@ vendor_compat.c ibm_tape.c hp_tape.c +quantum_tape.c diff --git a/src/tape_drivers/freebsd/cam/Makefile.am b/src/tape_drivers/freebsd/cam/Makefile.am index bea3527e..5498be6c 100644 --- a/src/tape_drivers/freebsd/cam/Makefile.am +++ b/src/tape_drivers/freebsd/cam/Makefile.am @@ -41,9 +41,9 @@ BASENAMES = libtape-cam AM_LIBTOOLFLAGS = --tag=disable-static libtape_cam_la_SOURCES = cam_cmn.c cam_tc.c vendor_compat.c ibm_tape.c hp_tape.c quantum_tape.c -libtape_cam_la_DEPENDENCIES = ../../../../messages/libtape_freebsd_cam_dat.a ../../../libltfs/libltfs.la libtape_cam_la-reed_solomon_crc.lo libtape_cam_la-crc32c_crc.lo libtape_cam_la-ibm_tape.lo -libtape_cam_la_LIBADD = ../../../libltfs/libltfs.la ./libtape_cam_la-reed_solomon_crc.lo ./libtape_cam_la-crc32c_crc.lo ./libtape_cam_la-ibm_tape.lo -libtape_cam_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../../../messages -ltape_freebsd_cam_dat +libtape_cam_la_DEPENDENCIES = ../../../../messages/libtape_freebsd_cam_dat.a ../../../libltfs/libltfs.la libtape_cam_la-reed_solomon_crc.lo libtape_cam_la-crc32c_crc.lo +libtape_cam_la_LIBADD = ../../../libltfs/libltfs.la ./libtape_cam_la-reed_solomon_crc.lo ./libtape_cam_la-crc32c_crc.lo +libtape_cam_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../../../messages/libtape_freebsd_cam_dat.a libtape_cam_la_CPPFLAGS = @AM_CPPFLAGS@ -I ../../.. -I ../.. vendor_compat.c: diff --git a/src/tape_drivers/freebsd/cam/cam_cmn.c b/src/tape_drivers/freebsd/cam/cam_cmn.c index 1b65df2e..6fef2b8e 100644 --- a/src/tape_drivers/freebsd/cam/cam_cmn.c +++ b/src/tape_drivers/freebsd/cam/cam_cmn.c @@ -775,6 +775,9 @@ int camtape_takedump_drive(void *device, bool nonforced_dump) struct camtape_data *softc = (struct camtape_data *)device; unsigned char *serial = softc->drive_serial; + if (softc->vendor != VENDOR_IBM) + return 0; + ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_TAKEDUMPDRV)); /* Make base filename */ time(&now); diff --git a/src/tape_drivers/freebsd/cam/cam_cmn.h b/src/tape_drivers/freebsd/cam/cam_cmn.h index f2e52066..f56fe1dd 100644 --- a/src/tape_drivers/freebsd/cam/cam_cmn.h +++ b/src/tape_drivers/freebsd/cam/cam_cmn.h @@ -382,6 +382,7 @@ struct camtape_data { bool loaded; /**< Is cartridge loaded? */ bool loadfailed; /**< Is load/unload failed? */ unsigned char drive_serial[255]; /**< serial number of device */ + int vendor; /**< drive vendor */ int drive_type; /**< device type */ int itd_command_size; /**< ITD sense conversion table size for commands */ struct itd_conversion_entry *itd_command; /**< ITD sense conversion table for commands */ @@ -450,8 +451,8 @@ extern int camtape_get_info(void *device, struct tc_drive_info *info); extern int camtape_set_profiler(void *device, char *work_dir, bool enable); extern int camtape_get_timeout(struct timeout_tape *table, int op_code); -extern int camtape_logsense_page(struct camtape_data *softc, const uint8_t page, const uint8_t subpage, - unsigned char *buf, const size_t size); +extern int camtape_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size); /** * Get Dump @@ -578,7 +579,7 @@ static inline bool is_dump_required_error(struct camtape_data *softc, int ret, b log_page.parm_pointer = 0; memset(log_page.data, 0, LOGSENSEPAGE); - rc = camtape_logsense_page(softc, log_page.page_code, 0, (unsigned char *)&log_page.data, + rc = camtape_logsense(softc, log_page.page_code, 0, (unsigned char *)&log_page.data, LOGSENSEPAGE); ans = (rc == DEVICE_GOOD); diff --git a/src/tape_drivers/freebsd/cam/cam_tc.c b/src/tape_drivers/freebsd/cam/cam_tc.c index 374bae7f..d5ae85ec 100644 --- a/src/tape_drivers/freebsd/cam/cam_tc.c +++ b/src/tape_drivers/freebsd/cam/cam_tc.c @@ -279,12 +279,11 @@ int camtape_open(const char *devname, void **handle) ltfsmsg(LTFS_INFO, 31229I, vendor); /* Check the drive is supportable */ - struct supported_device **cur = ibm_supported_drives; + struct supported_device **cur = get_supported_devs((char *)softc->cd->inq_data.vendor); while(*cur) { - if ((! strncmp((char*)softc->cd->inq_data.vendor, (*cur)->vendor_id, - strlen((*cur)->vendor_id)) ) && - (! strncmp((char*)softc->cd->inq_data.product, (*cur)->product_id, - strlen((*cur)->product_id)) ) ) { + if ((! strncmp((char*)softc->cd->inq_data.vendor, (*cur)->vendor_id, strlen((*cur)->vendor_id)) ) && + (! strncmp((char*)softc->cd->inq_data.product, (*cur)->product_id, strlen((*cur)->product_id)) ) ) { + softc->vendor = (*cur)->vendor_type; drive_type = (*cur)->drive_type; break; } @@ -294,12 +293,11 @@ int camtape_open(const char *devname, void **handle) if (drive_type != DRIVE_UNSUPPORTED) { softc->drive_type = drive_type; - /* Setup IBM tape specific parameters */ - standard_table = standard_tape_errors; - vendor_table = ibm_tape_errors; - - /* Set specific timeout value based on drive type */ - ibm_tape_init_timeout(&softc->timeouts, softc->drive_type); + /* Setup vendor specific parameters */ + init_error_table(softc->vendor, &standard_table, &vendor_table); + init_timeout(softc->vendor, &softc->timeouts, softc->drive_type); + if (!softc->timeouts) + ibm_tape_init_timeout(&softc->timeouts, softc->drive_type); } else { ltfsmsg(LTFS_INFO, 31230I, softc->cd->inq_data.product); close(softc->fd_sa); @@ -313,7 +311,7 @@ int camtape_open(const char *devname, void **handle) memcpy(softc->drive_serial, softc->cd->serial_num, softc->cd->serial_num_len); ltfsmsg(LTFS_INFO, 31232I, softc->cd->inq_data.revision); - if (! ibm_tape_is_supported_firmware(softc->drive_type, (uint8_t *)softc->cd->inq_data.revision)) { + if (! drive_has_supported_fw(softc->vendor, softc->drive_type, (uint8_t *)softc->cd->inq_data.revision)) { ltfsmsg(LTFS_INFO, 31230I, "firmware", softc->cd->inq_data.revision); close(softc->fd_sa); close_cd_pass_device(softc); @@ -1133,16 +1131,29 @@ int camtape_load(void *device, struct tc_position *pos) softc->force_readperm = DEFAULT_READPERM; softc->write_counter = 0; softc->read_counter = 0; - softc->cart_type = buf[2]; softc->density_code = buf[8]; + if (buf[2] == 0x00 || buf[2] == 0x01) { + /* + * Non-IBM drive doesn't have cartridge type so need to assume from density code. + */ + softc->cart_type = assume_cart_type(softc->density_code); + if (buf[2] == 0x01) + softc->is_worm = true; + } else { + /* + * IBM drive haves cartridge type in buf[2] like TC_MP_LTO5D_CART. + */ + softc->cart_type = buf[2]; + } + if (softc->cart_type == 0x00) { ltfsmsg(LTFS_WARN, 31253W); ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOAD)); return 0; } - rc = ibm_tape_is_supported_tape(softc->cart_type, softc->density_code, &(softc->is_worm)); + rc = is_supported_tape(softc->cart_type, softc->density_code, &(softc->is_worm)); if(rc == -LTFS_UNSUPPORTED_MEDIUM) ltfsmsg(LTFS_INFO, 31255I, softc->cart_type, softc->density_code); @@ -1185,10 +1196,10 @@ int camtape_unload(void *device, struct tc_position *pos) } /* - * Get the number of blocks in the buffer on the tape drive after a write. Eventually it would + * Get the first block position that is not transferred to the medium. Eventually it would * be nice to include this in status information returned from the sa(4) driver. */ -static int camtape_get_block_in_buffer(void *device, uint32_t *block) +static int camtape_get_next_block_to_xfer(void *device, struct tc_position *pos) { int rc; union ccb *ccb = NULL; @@ -1232,9 +1243,10 @@ static int camtape_get_block_in_buffer(void *device, uint32_t *block) if (rc != DEVICE_GOOD) camtape_process_errors(softc, rc, msg, "READPOS", true); else { - *block = scsi_3btoul(ext_data.num_objects); - ltfsmsg(LTFS_DEBUG, 30398D, "blocks-in-buffer", - (unsigned long long) *block, 0, 0, softc->drive_serial); + pos->partition = ext_data.partition; + pos->block = scsi_8btou64(ext_data.last_object); + ltfsmsg(LTFS_DEBUG, 30398D, "next-block-to-xfer", + (unsigned long long) pos->block, 0, 0, softc->drive_serial); } bailout: @@ -1673,8 +1685,8 @@ int camtape_format(void *device, TC_FORMAT_TYPE format, const char *vol_name, co */ #define MAX_UINT16 (0x0000FFFF) -int camtape_logsense_page(struct camtape_data *softc, const uint8_t page, const uint8_t subpage, - unsigned char *buf, const size_t size) +int camtape_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size) { int rc = DEVICE_GOOD; char *msg = NULL; @@ -1682,8 +1694,18 @@ int camtape_logsense_page(struct camtape_data *softc, const uint8_t page, const union ccb *ccb = NULL; int timeout; - ltfsmsg(LTFS_DEBUG3, 31397D, "logsense", (unsigned long long)page, (unsigned long long)subpage, - softc->drive_serial); + struct camtape_data *softc = device; + + unsigned int len = 0; + unsigned char *inner_buf = NULL; + + ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOGSENSE)); + ltfsmsg(LTFS_DEBUG3, 31397D, "logsense", + (unsigned long long)page, (unsigned long long)subpage, softc->drive_serial); + + inner_buf = calloc(1, MAXLP_SIZE); /* Assume max length of LP is 0xFFFF */ + if (!inner_buf) + return -LTFS_NO_MEMORY; ccb = cam_getccb(softc->cd); if (ccb == NULL) { @@ -1709,8 +1731,8 @@ int camtape_logsense_page(struct camtape_data *softc, const uint8_t page, const /*save_pages*/ 0, /*ppc*/ 0, /*paramptr*/ 0, - /*param_buf*/ buf, - /*param_len*/ size, + /*param_buf*/ inner_buf, + /*param_len*/ MAXLP_SIZE, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ timeout); /* @@ -1725,23 +1747,27 @@ int camtape_logsense_page(struct camtape_data *softc, const uint8_t page, const if (rc != DEVICE_GOOD) camtape_process_errors(softc, rc, msg, "logsense page", true); + else { + len = ((int)inner_buf[2] << 8) + (int)inner_buf[3] + 4; + + if (size > len) + memcpy(buf, inner_buf, len); + else + memcpy(buf, inner_buf, size); + + rc = len; + } bailout: + if (inner_buf != NULL) + free(inner_buf); + if (ccb != NULL) cam_freeccb(ccb); - return rc; -} - -int camtape_logsense(void *device, const uint8_t page, unsigned char *buf, const size_t size) -{ - struct camtape_data *softc = (struct camtape_data *)device; - int ret = 0; - - ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOGSENSE)); - ret = camtape_logsense_page(softc, page, 0, buf, size); ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); - return ret; + + return rc; } #define PARTITIOIN_REC_HEADER_LEN (4) @@ -1758,10 +1784,13 @@ int camtape_remaining_capacity(void *device, struct tc_remaining_cap *cap) int rc; ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_REMAINCAP)); - if (IS_LTO(softc->drive_type) && (DRIVE_GEN(softc->drive_type) == 0x05)) { + if ((IS_LTO(softc->drive_type) && (DRIVE_GEN(softc->drive_type) == 0x05)) || + (softc->vendor == VENDOR_HP && IS_LTO(softc->drive_type) && (DRIVE_GEN(softc->drive_type) == 0x06)) || + (softc->vendor == VENDOR_QUANTUM_B && IS_LTO(softc->drive_type))) { + /* Issue LogPage 0x31 */ - rc = camtape_logsense(device, LOG_TAPECAPACITY, logdata, LOGSENSEPAGE); - if (rc) { + rc = camtape_logsense(device, LOG_TAPECAPACITY, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) { ltfsmsg(LTFS_INFO, 31257I, LOG_TAPECAPACITY, rc); ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_REMAINCAP)); return rc; @@ -1800,8 +1829,8 @@ int camtape_remaining_capacity(void *device, struct tc_remaining_cap *cap) } else { /* Issue LogPage 0x17 */ - rc = camtape_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); - if (rc) { + rc = camtape_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) { ltfsmsg(LTFS_INFO, 31257I, LOG_VOLUMESTATS, rc); ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_REMAINCAP)); return rc; @@ -1961,7 +1990,7 @@ int camtape_modeselect(void *device, unsigned char *buf, const size_t size) /*retries*/ 1, /*cbfcnp*/ NULL, /*tag_action*/ MSG_SIMPLE_Q_TAG, - /*scsi_page_fmt*/ 0, + /*scsi_page_fmt*/ 1, /*save_pages*/ 0, /*param_buf*/ buf, /*param_len*/ size, @@ -2050,6 +2079,10 @@ int camtape_read_attribute(void *device, const tape_partition_t part, const uint size_t attr_size; union ccb *ccb = NULL; + /* TODO: Need to return data with header when size is MAXMAM_SIZE */ + if (size == MAXMAM_SIZE) + return -LTFS_NO_XATTR; + ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_READATTR)); ltfsmsg(LTFS_DEBUG3, 31397D, "readattr", (unsigned long long)part, (unsigned long long)id, softc->drive_serial); @@ -2117,6 +2150,7 @@ int camtape_read_attribute(void *device, const tape_partition_t part, const uint id != TC_MAM_APP_FORMAT_VERSION) ltfsmsg(LTFS_INFO, 31260I, rc); } else { + memcpy(buf, &attr_header[1], size); } @@ -2277,6 +2311,24 @@ int camtape_allow_overwrite(void *device, const struct tc_position pos) return rc; } +/** + * GRAO command is currently unsupported on this device + */ +int camtape_grao(void *device, unsigned char *buf, const uint32_t len) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + +/** + * RRAO command is currently unsupported on this device + */ +int camtape_rrao(void *device, unsigned char *buf, const uint32_t len, size_t *out_size) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + /** * Set compression setting * @param device a pointer to the camtape backend @@ -2363,13 +2415,18 @@ int camtape_set_default(void *device) } /* set logical block protection */ - if (global_data.crc_checking) { - ltfsmsg(LTFS_DEBUG, 31392D, __FUNCTION__, "Setting LBP"); - rc = camtape_set_lbp(device, true); + if (softc->vendor == VENDOR_IBM) { + if (global_data.crc_checking) { + ltfsmsg(LTFS_DEBUG, 31392D, __FUNCTION__, "Setting LBP"); + rc = camtape_set_lbp(device, true); + } else { + ltfsmsg(LTFS_DEBUG, 31392D, __FUNCTION__, "Resetting LBP"); + rc = camtape_set_lbp(device, false); + } } else { - ltfsmsg(LTFS_DEBUG, 31392D, __FUNCTION__, "Resetting LBP"); - rc = camtape_set_lbp(device, false); + rc = DEVICE_GOOD; } + if (rc != DEVICE_GOOD) goto bailout; @@ -2447,8 +2504,8 @@ int camtape_get_cartridge_health(void *device, struct tc_cartridge_health *cart_ /* Issue LogPage 0x37 */ cart_health->tape_efficiency = UNSUPPORTED_CARTRIDGE_HEALTH; - rc = camtape_logsense(device, LOG_PERFORMANCE, logdata, LOGSENSEPAGE); - if (rc) + rc = camtape_logsense(device, LOG_PERFORMANCE, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 31261I, LOG_PERFORMANCE, rc, "get cart health"); else { for(i = 0; i < (int)((sizeof(perfstats)/sizeof(perfstats[0]))); i++) { /* BEAM: loop doesn't iterate - Use loop for future enhancement. */ @@ -2498,8 +2555,9 @@ int camtape_get_cartridge_health(void *device, struct tc_cartridge_health *cart_ cart_health->read_mbytes = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_begin = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_middle = UNSUPPORTED_CARTRIDGE_HEALTH; - rc = camtape_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); - if (rc) + + rc = camtape_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 31261I, LOG_VOLUMESTATS, rc, "get cart health"); else { for(i = 0; i < (int)((sizeof(volstats)/sizeof(volstats[0]))); i++) { @@ -2595,10 +2653,11 @@ int camtape_get_tape_alert(void *device, uint64_t *tape_alert) ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_GETTAPEALT)); /* Issue LogPage 0x2E */ ta = 0; - rc = camtape_logsense(device, LOG_TAPE_ALERT, logdata, LOGSENSEPAGE); - if (rc) + rc = camtape_logsense(device, LOG_TAPE_ALERT, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 31261I, LOG_TAPE_ALERT, rc, "get tape alert"); else { + rc = 0; for(i = 1; i <= 64; i++) { if (parse_logPage(logdata, (uint16_t) i, ¶m_size, buf, 16) || param_size != sizeof(uint8_t)) { @@ -3045,8 +3104,8 @@ int camtape_get_eod_status(void *device, int part) ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_GETEODSTAT)); /* Issue LogPage 0x17 */ - rc = camtape_logsense(device, LOG_VOL_STATISTICS, logdata, LOGSENSEPAGE); - if (rc) { + rc = camtape_logsense(device, LOG_VOL_STATISTICS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) { ltfsmsg(LTFS_WARN, 31264W, LOG_VOL_STATISTICS, rc); ltfs_profiler_add_entry(softc->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_GETEODSTAT)); return EOD_UNKNOWN; @@ -3114,11 +3173,12 @@ int camtape_get_xattr(void *device, const char *name, char **buf) get_current_timespec(&now); if ( softc->fetch_sec_acq_loss_w == 0 || ((softc->fetch_sec_acq_loss_w + 60 < now.tv_sec) && softc->dirty_acq_loss_w)) { - rc = camtape_logsense_page(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, - logdata, LOGSENSEPAGE); - if (rc) + rc = camtape_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, + logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 31261I, LOG_PERFORMANCE, rc, "get xattr"); else { + rc = 0; if (parse_logPage(logdata, PERF_ACTIVE_CQ_LOSS_W, ¶m_size, logbuf, 16)) { ltfsmsg(LTFS_INFO, 31262I, LOG_PERFORMANCE, "get xattr"); rc = -LTFS_NO_XATTR; @@ -4093,6 +4153,8 @@ struct tape_ops camtape_drive_handler = { .write_attribute = camtape_write_attribute, .read_attribute = camtape_read_attribute, .allow_overwrite = camtape_allow_overwrite, + .grao = camtape_grao, + .rrao = camtape_rrao, // May be command combination .set_compression = camtape_set_compression, .set_default = camtape_set_default, @@ -4115,7 +4177,7 @@ struct tape_ops camtape_drive_handler = { .get_serialnumber = camtape_get_serialnumber, .get_info = camtape_get_info, .set_profiler = camtape_set_profiler, - .get_block_in_buffer = camtape_get_block_in_buffer, + .get_next_block_to_xfer = camtape_get_next_block_to_xfer, .is_readonly = camtape_is_readonly }; diff --git a/src/tape_drivers/generic/file/Makefile.am b/src/tape_drivers/generic/file/Makefile.am index 264fcd0e..661043d5 100644 --- a/src/tape_drivers/generic/file/Makefile.am +++ b/src/tape_drivers/generic/file/Makefile.am @@ -41,7 +41,7 @@ AM_LIBTOOLFLAGS = --tag=disable-static libtape_file_la_SOURCES = filedebug_tc.c filedebug_conf_tc.c ibm_tape.c libtape_file_la_DEPENDENCIES = ../../../../messages/libtape_generic_file_dat.a ../../../libltfs/libltfs.la libtape_file_la_LIBADD =../../../libltfs/libltfs.la -libtape_file_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../../../messages/ -ltape_generic_file_dat +libtape_file_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../../../messages/libtape_generic_file_dat.a libtape_file_la_CPPFLAGS = @AM_CPPFLAGS@ -I ../../.. ibm_tape.c: diff --git a/src/tape_drivers/generic/file/filedebug_tc.c b/src/tape_drivers/generic/file/filedebug_tc.c index 1909d74b..b0ad4056 100644 --- a/src/tape_drivers/generic/file/filedebug_tc.c +++ b/src/tape_drivers/generic/file/filedebug_tc.c @@ -291,7 +291,12 @@ static void emulate_seek_wait(struct filedebug_data *state, struct tc_position * dest->block % blocks_per_wrap : blocks_per_wrap - (dest->block % blocks_per_wrap); - uint64_t distance = llabs(target_dist_from_bot - current_dist_from_bot); + uint64_t distance; + if (target_dist_from_bot > current_dist_from_bot) + distance = target_dist_from_bot - current_dist_from_bot; + else + distance = current_dist_from_bot - target_dist_from_bot; + float cost = ((float) state->conf.eot_to_bot_sec / blocks_per_wrap) * (distance-1.0); time_t delay_us = 0; @@ -809,7 +814,8 @@ int filedebug_write(void *device, const char *buf, size_t count, struct tc_posit return -EDEV_WRITE_PERM; } else if ( state->write_counter > (state->force_writeperm - THRESHOLD_FORCE_WRITE_NO_WRITE) ) { ltfsmsg(LTFS_INFO, 30019I); - pos->block++; + ++state->current_position.block; + pos->block = state->current_position.block; return DEVICE_GOOD; } } @@ -1412,6 +1418,18 @@ static inline int _sanitize_tape(struct filedebug_data *state) ret = -EDEV_MEDIUM_FORMAT_ERROR; break; } + } else if (gen == DRIVE_GEN_JAG7) { + switch (state->conf.cart_type) { + case TC_MP_JF: + state->is_worm = false; + break; + default: + ltfsmsg(LTFS_INFO, 30086I, "TS1170", state->conf.cart_type); + state->is_worm = false; + state->unsupported_tape = true; + ret = -EDEV_MEDIUM_FORMAT_ERROR; + break; + } } else { ltfsmsg(LTFS_INFO, 30086I, "Unexpected Drive", state->conf.cart_type); state->is_worm = false; @@ -1867,8 +1885,7 @@ int filedebug_set_xattr(void *device, const char *name, const char *buf, size_t state->force_writeperm = perm_count; state->clear_by_pc = false; } - if (state->force_writeperm && state->force_writeperm < THRESHOLD_FORCE_WRITE_NO_WRITE) - state->force_writeperm = THRESHOLD_FORCE_WRITE_NO_WRITE; + state->write_counter = 0; ret = DEVICE_GOOD; } else if (! strcmp(name, "ltfs.vendor.IBM.forceErrorType")) { @@ -1901,7 +1918,7 @@ int filedebug_set_xattr(void *device, const char *name, const char *buf, size_t return ret; } -int filedebug_logsense(void *device, const uint8_t page, unsigned char *buf, const size_t size) +int filedebug_logsense(void *device, const uint8_t page, const uint8_t subpage, unsigned char *buf, const size_t size) { ltfsmsg(LTFS_ERR, 10007E, __FUNCTION__); return -EDEV_UNSUPPORTED_FUNCTION; @@ -2106,6 +2123,24 @@ int filedebug_allow_overwrite(void *device, const struct tc_position pos) return DEVICE_GOOD; } +/** + * GRAO command is currently unsupported on this device + */ +int filedebug_grao(void *device, unsigned char *buf, const uint32_t len) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + +/** + * RRAO command is currently unsupported on this device + */ +int filedebug_rrao(void *device, unsigned char *buf, const uint32_t len, size_t *out_size) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + int filedebug_get_eod_status(void *device, int partition) { struct filedebug_data *state = (struct filedebug_data *)device; @@ -2753,9 +2788,13 @@ int filedebug_set_profiler(void *device, char *work_dir, bool enable) return 0; } -int filedebug_get_block_in_buffer(void *device, unsigned int *block) +int filedebug_get_next_block_to_xfer(void *device, struct tc_position *pos) { - *block = 0; + struct filedebug_data *state = (struct filedebug_data *)device; + + pos->partition = state->current_position.partition; + pos->block = state->current_position.block - THRESHOLD_FORCE_WRITE_NO_WRITE; + return 0; } @@ -2791,6 +2830,8 @@ struct tape_ops filedebug_handler = { .write_attribute = filedebug_write_attribute, .read_attribute = filedebug_read_attribute, .allow_overwrite = filedebug_allow_overwrite, + .grao = filedebug_grao, + .rrao = filedebug_rrao, .set_compression = filedebug_set_compression, .set_default = filedebug_set_default, .get_cartridge_health = filedebug_get_cartridge_health, @@ -2812,7 +2853,7 @@ struct tape_ops filedebug_handler = { .get_serialnumber = filedebug_get_serialnumber, .get_info = filedebug_get_info, .set_profiler = filedebug_set_profiler, - .get_block_in_buffer = filedebug_get_block_in_buffer, + .get_next_block_to_xfer = filedebug_get_next_block_to_xfer, .is_readonly = filedebug_is_readonly, }; diff --git a/src/tape_drivers/generic/itdtimg/Makefile.am b/src/tape_drivers/generic/itdtimg/Makefile.am index 1504efb0..655eeb37 100644 --- a/src/tape_drivers/generic/itdtimg/Makefile.am +++ b/src/tape_drivers/generic/itdtimg/Makefile.am @@ -41,7 +41,7 @@ AM_LIBTOOLFLAGS = --tag=disable-static libtape_itdtimg_la_SOURCES = itdtimg_tc.c libtape_itdtimg_la_DEPENDENCIES = ../../../../messages/libtape_generic_itdtimg_dat.a ../../../libltfs/libltfs.la libtape_itdtimg_la_LIBADD = ../../../libltfs/libltfs.la -libtape_itdtimg_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../../../messages -ltape_generic_itdtimg_dat +libtape_itdtimg_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../../../messages/libtape_generic_itdtimg_dat.a libtape_itdtimg_la_CPPFLAGS = @AM_CPPFLAGS@ -I ../../.. install-exec-hook: diff --git a/src/tape_drivers/generic/itdtimg/itdtimg_tc.c b/src/tape_drivers/generic/itdtimg/itdtimg_tc.c index d739621e..133df2c0 100644 --- a/src/tape_drivers/generic/itdtimg/itdtimg_tc.c +++ b/src/tape_drivers/generic/itdtimg/itdtimg_tc.c @@ -857,7 +857,7 @@ int itdtimage_set_xattr(void *device, const char *name, const char *buf, size_t return -LTFS_NO_XATTR; } -int itdtimage_logsense(void *device, const uint8_t page, unsigned char *buf, const size_t size) +int itdtimage_logsense(void *device, const uint8_t page, const uint8_t subpage, unsigned char *buf, const size_t size) { ltfsmsg(LTFS_ERR, 10007E, __FUNCTION__); return -EDEV_UNSUPPORTED_FUNCTION; @@ -962,6 +962,24 @@ int itdtimage_allow_overwrite(void *device, const struct tc_position pos) return DEVICE_GOOD; } +/** + * GRAO command is currently unsupported on this device + */ +int itdtimage_grao(void *device, unsigned char *buf, const uint32_t len) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + +/** + * RRAO command is currently unsupported on this device + */ +int itdtimage_rrao(void *device, unsigned char *buf, const uint32_t len, size_t *out_size) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + int itdtimage_get_eod_status(void *vstate, int partition) { struct itdtimage_data *state = (struct itdtimage_data *)vstate; @@ -1351,9 +1369,9 @@ int itdtimage_get_device_list(struct tc_drive_info *buf, int count) if (buf && deventries < count) { snprintf(buf[deventries].name, TAPE_DEVNAME_LEN_MAX, "%s/%s", devdir, entry->d_name); - snprintf(buf[deventries].vendor, TAPE_VENDOR_NAME_LEN_MAX, "DUMMY"); - snprintf(buf[deventries].model, TAPE_MODEL_NAME_LEN_MAX, "DUMMYDEV"); - snprintf(buf[deventries].serial_number, TAPE_SERIAL_LEN_MAX, "%s", &(entry->d_name[strlen(DRIVE_FILE_PREFIX)])); + strncpy(buf[deventries].vendor, "DUMMY", TAPE_VENDOR_NAME_LEN_MAX); + strncpy(buf[deventries].model, "DUMMYDEV", TAPE_MODEL_NAME_LEN_MAX); + strncpy(buf[deventries].serial_number, &(entry->d_name[strlen(DRIVE_FILE_PREFIX)]), TAPE_SERIAL_LEN_MAX); ltfsmsg(LTFS_DEBUG, 31030D, buf[deventries].name, buf[deventries].vendor, buf[deventries].model, buf[deventries].serial_number); } @@ -1419,7 +1437,7 @@ int itdtimage_get_serialnumber(void *vstate, char **result) return DEVICE_GOOD; } -int filedebug_get_info(void *device, struct tc_drive_info *info) +int itdtimage_get_info(void *device, struct tc_drive_info *info) { /* * Return dummy data. @@ -1439,10 +1457,10 @@ int itdtimage_set_profiler(void *device, char *work_dir, bool enable) return 0; } -int itdtimage_get_block_in_buffer(void *device, unsigned int *block) +int itdtimage_get_next_block_to_xfer(void *device, struct tc_position *pos) { - *block = 0; - return 0; + /* This backend never accept write command */ + return -EDEV_WRITE_PROTECTED;; } /* Local functions */ @@ -1623,6 +1641,8 @@ struct tape_ops itdtimage_handler = { .write_attribute = itdtimage_write_attribute, .read_attribute = itdtimage_read_attribute, .allow_overwrite = itdtimage_allow_overwrite, + .grao = itdtimage_grao, + .rrao = itdtimage_rrao, .set_compression = itdtimage_set_compression, .set_default = itdtimage_set_default, .get_cartridge_health = itdtimage_get_cartridge_health, @@ -1642,8 +1662,9 @@ struct tape_ops itdtimage_handler = { .is_mountable = itdtimage_is_mountable, .get_worm_status = itdtimage_get_worm_status, .get_serialnumber = itdtimage_get_serialnumber, + .get_info = itdtimage_get_info, .set_profiler = itdtimage_set_profiler, - .get_block_in_buffer = itdtimage_get_block_in_buffer, + .get_next_block_to_xfer = itdtimage_get_next_block_to_xfer, .is_readonly = itdtimage_is_readonly, }; diff --git a/src/tape_drivers/hp_tape.c b/src/tape_drivers/hp_tape.c index ad9fd496..365ce4a4 100644 --- a/src/tape_drivers/hp_tape.c +++ b/src/tape_drivers/hp_tape.c @@ -63,10 +63,12 @@ #include "libltfs/ltfs_endian.h" struct supported_device *hp_supported_drives[] = { - TAPEDRIVE( HP_VENDOR_ID, "Ultrium 5-SCSI", DRIVE_LTO5, "[Ultrium 5-SCSI]" ), /* HP Ultrium Gen 5 */ - TAPEDRIVE( HP_VENDOR_ID, "Ultrium 6-SCSI", DRIVE_LTO6, "[Ultrium 6-SCSI]" ), /* HP Ultrium Gen 6 */ - TAPEDRIVE( HP_VENDOR_ID, "Ultrium 7-SCSI", DRIVE_LTO7, "[Ultrium 7-SCSI]" ), /* HP Ultrium Gen 7 */ - TAPEDRIVE( HPE_VENDOR_ID, "Ultrium 8-SCSI", DRIVE_LTO8, "[Ultrium 8-SCSI]" ), /* HPE Ultrium Gen 8 */ + TAPEDRIVE( HP_VENDOR_ID, "Ultrium 5-SCSI", VENDOR_HP, DRIVE_LTO5, "[Ultrium 5-SCSI]" ), /* HP Ultrium Gen 5 */ + TAPEDRIVE( HP_VENDOR_ID, "Ultrium 6-SCSI", VENDOR_HP, DRIVE_LTO6, "[Ultrium 6-SCSI]" ), /* HP Ultrium Gen 6 */ + TAPEDRIVE( HP_VENDOR_ID, "Ultrium 7-SCSI", VENDOR_HP, DRIVE_LTO7, "[Ultrium 7-SCSI]" ), /* HP Ultrium Gen 7 */ + TAPEDRIVE( HPE_VENDOR_ID, "Ultrium 8-SCSI", VENDOR_HP, DRIVE_LTO8, "[Ultrium 8-SCSI]" ), /* HPE Ultrium Gen 8 */ + TAPEDRIVE( TANDBERG_VENDOR_ID, "LTO-5 HH", VENDOR_HP, DRIVE_LTO5_HH, "[TANDBERG LTO5]" ), /* TANDBERG LTO-5 HH */ + TAPEDRIVE( TANDBERG_VENDOR_ID, "LTO-6 HH", VENDOR_HP, DRIVE_LTO6_HH, "[TANDBERG LTO6]" ), /* TANDBERG LTO-6 HH */ /* End of supported_devices */ NULL }; diff --git a/src/tape_drivers/hp_tape.h b/src/tape_drivers/hp_tape.h index 42515110..5af848ec 100644 --- a/src/tape_drivers/hp_tape.h +++ b/src/tape_drivers/hp_tape.h @@ -65,8 +65,9 @@ extern "C" { #endif -#define HP_VENDOR_ID "HP" -#define HPE_VENDOR_ID "HPE" +#define HP_VENDOR_ID "HP" +#define HPE_VENDOR_ID "HPE" +#define TANDBERG_VENDOR_ID "TANDBERG" extern struct error_table hp_tape_errors[]; diff --git a/src/tape_drivers/ibm_tape.c b/src/tape_drivers/ibm_tape.c index 88d7f576..e618f91e 100644 --- a/src/tape_drivers/ibm_tape.c +++ b/src/tape_drivers/ibm_tape.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2023 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -64,6 +64,10 @@ #include "libltfs/ltfs_endian.h" DRIVE_DENSITY_SUPPORT_MAP jaguar_drive_density[] = { + /* TS1170 */ + { DRIVE_GEN_JAG7, TC_MP_JF, TC_DC_JAG7, MEDIUM_PERFECT_MATCH }, + { DRIVE_GEN_JAG7, TC_MP_JF, TC_DC_UNKNOWN, MEDIUM_PROBABLY_WRITABLE }, + /* TS1160 */ { DRIVE_GEN_JAG6, TC_MP_JE, TC_DC_JAG6, MEDIUM_PERFECT_MATCH }, { DRIVE_GEN_JAG6, TC_MP_JE, TC_DC_UNKNOWN, MEDIUM_PROBABLY_WRITABLE }, @@ -141,6 +145,10 @@ DRIVE_DENSITY_SUPPORT_MAP jaguar_drive_density[] = { }; DRIVE_DENSITY_SUPPORT_MAP jaguar_drive_density_strict[] = { + /* TS1170 */ + { DRIVE_GEN_JAG7, TC_MP_JF, TC_DC_JAG7, MEDIUM_PERFECT_MATCH }, + { DRIVE_GEN_JAG7, TC_MP_JF, TC_DC_UNKNOWN, MEDIUM_PROBABLY_WRITABLE }, + /* TS1160 */ { DRIVE_GEN_JAG6, TC_MP_JE, TC_DC_JAG6, MEDIUM_PERFECT_MATCH }, { DRIVE_GEN_JAG6, TC_MP_JE, TC_DC_UNKNOWN, MEDIUM_PROBABLY_WRITABLE }, @@ -256,13 +264,16 @@ const unsigned char supported_cart[] = { TC_MP_JE, TC_MP_JV, TC_MP_JM, + TC_MP_JF, }; const unsigned char supported_density[] = { + TC_DC_JAG7E, TC_DC_JAG6E, TC_DC_JAG5AE, TC_DC_JAG5E, TC_DC_JAG4E, + TC_DC_JAG7, TC_DC_JAG6, TC_DC_JAG5A, TC_DC_JAG5, @@ -283,58 +294,60 @@ int num_supported_cart = sizeof(supported_cart)/sizeof(supported_ca int num_supported_density = sizeof(supported_density)/sizeof(supported_density[0]); struct supported_device *ibm_supported_drives[] = { - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD5", DRIVE_LTO5, "[ULTRIUM-TD5]" ), /* IBM Ultrium Gen 5 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD5", DRIVE_LTO5, "[ULT3580-TD5]" ), /* IBM Ultrium Gen 5 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH5", DRIVE_LTO5_HH, "[ULTRIUM-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH5", DRIVE_LTO5_HH, "[ULT3580-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 5", DRIVE_LTO5_HH, "[HH LTO Gen 5]" ), /* IBM Ultrium Gen 5 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD6", DRIVE_LTO6, "[ULTRIUM-TD6]" ), /* IBM Ultrium Gen 6 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD6", DRIVE_LTO6, "[ULT3580-TD6]" ), /* IBM Ultrium Gen 6 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH6", DRIVE_LTO6_HH, "[ULTRIUM-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH6", DRIVE_LTO6_HH, "[ULT3580-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 6", DRIVE_LTO6_HH, "[HH LTO Gen 6]" ), /* IBM Ultrium Gen 6 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD7", DRIVE_LTO7, "[ULTRIUM-TD7]" ), /* IBM Ultrium Gen 7 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD7", DRIVE_LTO7, "[ULT3580-TD7]" ), /* IBM Ultrium Gen 7 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH7", DRIVE_LTO7_HH, "[ULTRIUM-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH7", DRIVE_LTO7_HH, "[ULT3580-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 7", DRIVE_LTO7_HH, "[HH LTO Gen 7]" ), /* IBM Ultrium Gen 7 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD8", DRIVE_LTO8, "[ULTRIUM-TD8]" ), /* IBM Ultrium Gen 8 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD8", DRIVE_LTO8, "[ULT3580-TD8]" ), /* IBM Ultrium Gen 8 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH8", DRIVE_LTO8_HH, "[ULTRIUM-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH8", DRIVE_LTO8_HH, "[ULT3580-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 8", DRIVE_LTO8_HH, "[HH LTO Gen 8]" ), /* IBM Ultrium Gen 8 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD9", DRIVE_LTO9, "[ULTRIUM-TD9]" ), /* IBM Ultrium Gen 9 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD9", DRIVE_LTO9, "[ULT3580-TD9]" ), /* IBM Ultrium Gen 9 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH9", DRIVE_LTO9_HH, "[ULTRIUM-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH9", DRIVE_LTO9_HH, "[ULT3580-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 9", DRIVE_LTO9_HH, "[HH LTO Gen 9]" ), /* IBM Ultrium Gen 9 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "03592E07", DRIVE_TS1140, "[03592E07]" ), /* IBM TS1140 */ - TAPEDRIVE( IBM_VENDOR_ID, "03592E08", DRIVE_TS1150, "[03592E08]" ), /* IBM TS1150 */ - TAPEDRIVE( IBM_VENDOR_ID, "0359255F", DRIVE_TS1155, "[0359255F]" ), /* IBM TS1155 */ - TAPEDRIVE( IBM_VENDOR_ID, "0359255E", DRIVE_TS1155, "[0359255E]" ), /* IBM TS1155 */ - TAPEDRIVE( IBM_VENDOR_ID, "0359260F", DRIVE_TS1160, "[0359260F]" ), /* IBM TS1160 */ - TAPEDRIVE( IBM_VENDOR_ID, "0359260E", DRIVE_TS1160, "[0359260E]" ), /* IBM TS1160 */ - TAPEDRIVE( IBM_VENDOR_ID, "0359260S", DRIVE_TS1160, "[0359260S]" ), /* IBM TS1160 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD5", VENDOR_IBM, DRIVE_LTO5, "[ULTRIUM-TD5]" ), /* IBM Ultrium Gen 5 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD5", VENDOR_IBM, DRIVE_LTO5, "[ULT3580-TD5]" ), /* IBM Ultrium Gen 5 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH5", VENDOR_IBM, DRIVE_LTO5_HH, "[ULTRIUM-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH5", VENDOR_IBM, DRIVE_LTO5_HH, "[ULT3580-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 5", VENDOR_IBM, DRIVE_LTO5_HH, "[HH LTO Gen 5]" ), /* IBM Ultrium Gen 5 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD6", VENDOR_IBM, DRIVE_LTO6, "[ULTRIUM-TD6]" ), /* IBM Ultrium Gen 6 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD6", VENDOR_IBM, DRIVE_LTO6, "[ULT3580-TD6]" ), /* IBM Ultrium Gen 6 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH6", VENDOR_IBM, DRIVE_LTO6_HH, "[ULTRIUM-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH6", VENDOR_IBM, DRIVE_LTO6_HH, "[ULT3580-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 6", VENDOR_IBM, DRIVE_LTO6_HH, "[HH LTO Gen 6]" ), /* IBM Ultrium Gen 6 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD7", VENDOR_IBM, DRIVE_LTO7, "[ULTRIUM-TD7]" ), /* IBM Ultrium Gen 7 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD7", VENDOR_IBM, DRIVE_LTO7, "[ULT3580-TD7]" ), /* IBM Ultrium Gen 7 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH7", VENDOR_IBM, DRIVE_LTO7_HH, "[ULTRIUM-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH7", VENDOR_IBM, DRIVE_LTO7_HH, "[ULT3580-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 7", VENDOR_IBM, DRIVE_LTO7_HH, "[HH LTO Gen 7]" ), /* IBM Ultrium Gen 7 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD8", VENDOR_IBM, DRIVE_LTO8, "[ULTRIUM-TD8]" ), /* IBM Ultrium Gen 8 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD8", VENDOR_IBM, DRIVE_LTO8, "[ULT3580-TD8]" ), /* IBM Ultrium Gen 8 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH8", VENDOR_IBM, DRIVE_LTO8_HH, "[ULTRIUM-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH8", VENDOR_IBM, DRIVE_LTO8_HH, "[ULT3580-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 8", VENDOR_IBM, DRIVE_LTO8_HH, "[HH LTO Gen 8]" ), /* IBM Ultrium Gen 8 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-TD9", VENDOR_IBM, DRIVE_LTO9, "[ULTRIUM-TD9]" ), /* IBM Ultrium Gen 9 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD9", VENDOR_IBM, DRIVE_LTO9, "[ULT3580-TD9]" ), /* IBM Ultrium Gen 9 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH9", VENDOR_IBM, DRIVE_LTO9_HH, "[ULTRIUM-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH9", VENDOR_IBM, DRIVE_LTO9_HH, "[ULT3580-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "HH LTO Gen 9", VENDOR_IBM, DRIVE_LTO9_HH, "[HH LTO Gen 9]" ), /* IBM Ultrium Gen 9 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "03592E07", VENDOR_IBM, DRIVE_TS1140, "[03592E07]" ), /* IBM TS1140 */ + TAPEDRIVE( IBM_VENDOR_ID, "03592E08", VENDOR_IBM, DRIVE_TS1150, "[03592E08]" ), /* IBM TS1150 */ + TAPEDRIVE( IBM_VENDOR_ID, "0359255F", VENDOR_IBM, DRIVE_TS1155, "[0359255F]" ), /* IBM TS1155 */ + TAPEDRIVE( IBM_VENDOR_ID, "0359255E", VENDOR_IBM, DRIVE_TS1155, "[0359255E]" ), /* IBM TS1155 */ + TAPEDRIVE( IBM_VENDOR_ID, "0359260F", VENDOR_IBM, DRIVE_TS1160, "[0359260F]" ), /* IBM TS1160 */ + TAPEDRIVE( IBM_VENDOR_ID, "0359260E", VENDOR_IBM, DRIVE_TS1160, "[0359260E]" ), /* IBM TS1160 */ + TAPEDRIVE( IBM_VENDOR_ID, "0359260S", VENDOR_IBM, DRIVE_TS1160, "[0359260S]" ), /* IBM TS1160 */ + TAPEDRIVE( IBM_VENDOR_ID, "0359270F", VENDOR_IBM, DRIVE_TS1170, "[0359270F]" ), /* IBM TS1170 */ + TAPEDRIVE( IBM_VENDOR_ID, "0359270S", VENDOR_IBM, DRIVE_TS1170, "[0359270S]" ), /* IBM TS1170 */ /* End of supported_devices */ NULL }; struct supported_device *usb_supported_drives[] = { - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD5", DRIVE_LTO5, "[ULT3580-TD5]" ), /* IBM Ultrium Gen 5 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH5", DRIVE_LTO5_HH, "[ULTRIUM-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH5", DRIVE_LTO5_HH, "[ULT3580-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD6", DRIVE_LTO6, "[ULT3580-TD6]" ), /* IBM Ultrium Gen 6 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH6", DRIVE_LTO6_HH, "[ULTRIUM-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH6", DRIVE_LTO6_HH, "[ULT3580-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD7", DRIVE_LTO7, "[ULT3580-TD7]" ), /* IBM Ultrium Gen 7 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH7", DRIVE_LTO7_HH, "[ULTRIUM-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH7", DRIVE_LTO7_HH, "[ULT3580-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD8", DRIVE_LTO8, "[ULT3580-TD8]" ), /* IBM Ultrium Gen 8 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH8", DRIVE_LTO8_HH, "[ULTRIUM-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH8", DRIVE_LTO8_HH, "[ULT3580-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD9", DRIVE_LTO9, "[ULT3580-TD9]" ), /* IBM Ultrium Gen 9 */ - TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH9", DRIVE_LTO9_HH, "[ULTRIUM-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ - TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH9", DRIVE_LTO9_HH, "[ULT3580-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD5", VENDOR_IBM, DRIVE_LTO5, "[ULT3580-TD5]" ), /* IBM Ultrium Gen 5 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH5", VENDOR_IBM, DRIVE_LTO5_HH, "[ULTRIUM-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH5", VENDOR_IBM, DRIVE_LTO5_HH, "[ULT3580-HH5]" ), /* IBM Ultrium Gen 5 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD6", VENDOR_IBM, DRIVE_LTO6, "[ULT3580-TD6]" ), /* IBM Ultrium Gen 6 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH6", VENDOR_IBM, DRIVE_LTO6_HH, "[ULTRIUM-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH6", VENDOR_IBM, DRIVE_LTO6_HH, "[ULT3580-HH6]" ), /* IBM Ultrium Gen 6 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD7", VENDOR_IBM, DRIVE_LTO7, "[ULT3580-TD7]" ), /* IBM Ultrium Gen 7 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH7", VENDOR_IBM, DRIVE_LTO7_HH, "[ULTRIUM-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH7", VENDOR_IBM, DRIVE_LTO7_HH, "[ULT3580-HH7]" ), /* IBM Ultrium Gen 7 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD8", VENDOR_IBM, DRIVE_LTO8, "[ULT3580-TD8]" ), /* IBM Ultrium Gen 8 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH8", VENDOR_IBM, DRIVE_LTO8_HH, "[ULTRIUM-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH8", VENDOR_IBM, DRIVE_LTO8_HH, "[ULT3580-HH8]" ), /* IBM Ultrium Gen 8 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-TD9", VENDOR_IBM, DRIVE_LTO9, "[ULT3580-TD9]" ), /* IBM Ultrium Gen 9 */ + TAPEDRIVE( IBM_VENDOR_ID, "ULTRIUM-HH9", VENDOR_IBM, DRIVE_LTO9_HH, "[ULTRIUM-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ + TAPEDRIVE( IBM_VENDOR_ID, "ULT3580-HH9", VENDOR_IBM, DRIVE_LTO9_HH, "[ULT3580-HH9]" ), /* IBM Ultrium Gen 9 Half-High */ /* End of supported_devices */ NULL }; @@ -500,8 +513,8 @@ static struct _timeout_tape timeout_lto6[] = { { ERASE, 24600 }, { FORMAT_MEDIUM, 3000 }, { LOAD_UNLOAD, 780 }, - { LOCATE10, 2040 }, - { LOCATE16, 2040 }, + { LOCATE10, 2940 }, + { LOCATE16, 2940 }, { READ, 1500 }, { READ_BUFFER, 480 }, { REWIND, 600 }, @@ -519,7 +532,7 @@ static struct _timeout_tape timeout_lto6[] = { static struct _timeout_tape timeout_lto7[] = { { ERASE, 27540 }, { FORMAT_MEDIUM, 3000 }, - { LOAD_UNLOAD, 780 }, + { LOAD_UNLOAD, 960 }, { LOCATE10, 2880 }, { LOCATE16, 2880 }, { READ, 2280 }, @@ -537,9 +550,9 @@ static struct _timeout_tape timeout_lto7[] = { }; static struct _timeout_tape timeout_lto8[] = { - { ERASE, 46380 }, + { ERASE, 54896 }, { FORMAT_MEDIUM, 3000 }, - { LOAD_UNLOAD, 780 }, + { LOAD_UNLOAD, 960 }, { LOCATE10, 2880 }, { LOCATE16, 2880 }, { READ, 2280 }, @@ -549,7 +562,7 @@ static struct _timeout_tape timeout_lto8[] = { { SET_CAPACITY, 780 }, { SPACE6, 2880 }, { SPACE16, 2880 }, - { VERIFY, 28860 }, + { VERIFY, 47700 }, { WRITE, 1500 }, { WRITE_BUFFER, 540 }, { WRITE_FILEMARKS6, 1620 }, @@ -557,19 +570,19 @@ static struct _timeout_tape timeout_lto8[] = { }; static struct _timeout_tape timeout_lto9[] = { - { ERASE, 46380 }, + { ERASE, 74341 }, { FORMAT_MEDIUM, 3000 }, - { LOAD_UNLOAD, 780 }, - { LOCATE10, 2880 }, - { LOCATE16, 2880 }, - { READ, 2280 }, + { LOAD_UNLOAD, 960 }, + { LOCATE10, 2940 }, + { LOCATE16, 2940 }, + { READ, 2340 }, { READ_BUFFER, 480 }, { REWIND, 600 }, { SEND_DIAGNOSTIC, 1980 }, { SET_CAPACITY, 780 }, - { SPACE6, 2880 }, - { SPACE16, 2880 }, - { VERIFY, 28860 }, + { SPACE6, 2940 }, + { SPACE16, 2940 }, + { VERIFY, 63300 }, { WRITE, 1500 }, { WRITE_BUFFER, 540 }, { WRITE_FILEMARKS6, 1620 }, @@ -619,12 +632,12 @@ static struct _timeout_tape timeout_lto6_hh[] = { static struct _timeout_tape timeout_lto7_hh[] = { { ERASE, 27540 }, { FORMAT_MEDIUM, 3240 }, - { LOAD_UNLOAD, 840 }, + { LOAD_UNLOAD, 960 }, { LOCATE10, 2940 }, { LOCATE16, 2940 }, { READ, 2340 }, { READ_BUFFER, 480 }, - { REWIND, 660 }, + { REWIND, 600 }, { SEND_DIAGNOSTIC, 2040 }, { SET_CAPACITY, 960 }, { SPACE6, 2940 }, @@ -637,42 +650,42 @@ static struct _timeout_tape timeout_lto7_hh[] = { }; static struct _timeout_tape timeout_lto8_hh[] = { - { ERASE, 46380 }, - { FORMAT_MEDIUM, 3240 }, - { LOAD_UNLOAD, 840 }, - { LOCATE10, 2940 }, - { LOCATE16, 2940 }, - { READ, 2340 }, - { READ_BUFFER, 480 }, - { REWIND, 660 }, - { SEND_DIAGNOSTIC, 2040 }, - { SET_CAPACITY, 960 }, - { SPACE6, 2940 }, - { SPACE16, 2940 }, - { VERIFY, 47700 }, - { WRITE, 1560 }, - { WRITE_BUFFER, 540 }, - { WRITE_FILEMARKS6, 1680 }, + { ERASE, 121448 }, + { FORMAT_MEDIUM, 3240 }, + { LOAD_UNLOAD, 960 }, + { LOCATE10, 2940 }, + { LOCATE16, 2940 }, + { READ, 2340 }, + { READ_BUFFER, 480 }, + { REWIND, 600 }, + { SEND_DIAGNOSTIC, 2040 }, + { SET_CAPACITY, 960 }, + { SPACE6, 2940 }, + { SPACE16, 2940 }, + { VERIFY, 54360 }, + { WRITE, 1560 }, + { WRITE_BUFFER, 540 }, + { WRITE_FILEMARKS6, 1680 }, {-1, -1} }; static struct _timeout_tape timeout_lto9_hh[] = { - { ERASE, 46380 }, - { FORMAT_MEDIUM, 3240 }, - { LOAD_UNLOAD, 840 }, - { LOCATE10, 2940 }, - { LOCATE16, 2940 }, - { READ, 2340 }, - { READ_BUFFER, 480 }, - { REWIND, 660 }, - { SEND_DIAGNOSTIC, 2040 }, - { SET_CAPACITY, 960 }, - { SPACE6, 2940 }, - { SPACE16, 2940 }, - { VERIFY, 47700 }, - { WRITE, 1560 }, - { WRITE_BUFFER, 540 }, - { WRITE_FILEMARKS6, 1680 }, + { ERASE, 166370 }, + { FORMAT_MEDIUM, 3240 }, + { LOAD_UNLOAD, 960 }, + { LOCATE10, 2940 }, + { LOCATE16, 2940 }, + { READ, 2340 }, + { READ_BUFFER, 480 }, + { REWIND, 600 }, + { SEND_DIAGNOSTIC, 2040 }, + { SET_CAPACITY, 960 }, + { SPACE6, 2940 }, + { SPACE16, 2940 }, + { VERIFY, 63300 }, + { WRITE, 1560 }, + { WRITE_BUFFER, 540 }, + { WRITE_FILEMARKS6, 1680 }, {-1, -1} }; @@ -794,6 +807,26 @@ static struct _timeout_tape timeout_1160[] = { {-1, -1} }; +static struct _timeout_tape timeout_1170[] = { + { XCOPY, 176820 }, + { ERASE, 175900 }, + { FORMAT_MEDIUM, 3120 }, + { LOAD_UNLOAD, 900 }, + { LOCATE10, 2280 }, + { LOCATE16, 2240 }, + { READ, 2340 }, + { READ_BUFFER, 480 }, + { REWIND, 600 }, + { SEND_DIAGNOSTIC, 2280 }, + { SPACE6, 2280 }, + { SPACE16, 2240 }, + { VERIFY, 176820 }, + { WRITE, 1440 }, + { WRITE_BUFFER, 540 }, + { WRITE_FILEMARKS6, 1380 }, + {-1, -1} +}; + static int _create_table_tape(struct timeout_tape **result, struct _timeout_tape* base, struct _timeout_tape* override) @@ -882,6 +915,9 @@ int ibm_tape_init_timeout(struct timeout_tape** table, int type) case DRIVE_TS1160: ret = _create_table_tape(table, timeout_11x0, timeout_1160); break; + case DRIVE_TS1170: + ret = _create_table_tape(table, timeout_11x0, timeout_1170); + break; default: ret = _create_table_tape(table, timeout_lto, timeout_lto7_hh); break; @@ -930,6 +966,9 @@ static inline unsigned char _assume_cartridge_type(char product, char btype) case 'M': ctype = TC_MP_JM; break; + case 'F': + ctype = TC_MP_JF; + break; default: break; } @@ -1033,6 +1072,9 @@ char* ibm_tape_assume_cart_name(unsigned char type) case TC_MP_JM: name = "JM"; break; + case TC_MP_JF: + name = "JF"; + break; default: name = "L5"; break; @@ -1295,6 +1337,7 @@ bool ibm_tape_is_supported_firmware(int drive_type, const unsigned char * const case DRIVE_LTO7_HH: case DRIVE_TS1150: case DRIVE_TS1160: + case DRIVE_TS1170: default: break; } diff --git a/src/tape_drivers/linux/lin_tape/Makefile.am b/src/tape_drivers/linux/lin_tape/Makefile.am index 608270b0..57335706 100644 --- a/src/tape_drivers/linux/lin_tape/Makefile.am +++ b/src/tape_drivers/linux/lin_tape/Makefile.am @@ -41,7 +41,7 @@ AM_LIBTOOLFLAGS = --tag=disable-static libtape_lin_tape_la_SOURCES = lin_tape_ibmtape.c vendor_compat.c ibm_tape.c hp_tape.c quantum_tape.c libtape_lin_tape_la_DEPENDENCIES = ../../../../messages/libtape_linux_lin_tape_dat.a ../../../libltfs/libltfs.la ./libtape_lin_tape_la-reed_solomon_crc.lo ./libtape_lin_tape_la-crc32c_crc.lo libtape_lin_tape_la_LIBADD = ../../../libltfs/libltfs.la ./libtape_lin_tape_la-reed_solomon_crc.lo ./libtape_lin_tape_la-crc32c_crc.lo -libtape_lin_tape_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../../../messages -ltape_linux_lin_tape_dat +libtape_lin_tape_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../../../messages/libtape_linux_lin_tape_dat.a libtape_lin_tape_la_CPPFLAGS = @AM_CPPFLAGS@ -I ../../.. -I ../.. vendor_compat.c: diff --git a/src/tape_drivers/linux/lin_tape/lin_tape_ibmtape.c b/src/tape_drivers/linux/lin_tape/lin_tape_ibmtape.c index 61e673c8..b232626f 100644 --- a/src/tape_drivers/linux/lin_tape/lin_tape_ibmtape.c +++ b/src/tape_drivers/linux/lin_tape/lin_tape_ibmtape.c @@ -1056,7 +1056,7 @@ int lin_tape_ibmtape_open(const char *devname, void **handle) priv->fd = open(devfile, O_RDWR | O_NDELAY); if (priv->fd < 0) { - priv->fd = open(devname, O_RDONLY | O_NDELAY); + priv->fd = open(devfile, O_RDONLY | O_NDELAY); if (priv->fd < 0) { if (errno == EAGAIN) { ltfsmsg(LTFS_ERR, 30424E, devname); @@ -2067,7 +2067,7 @@ int lin_tape_ibmtape_unload(void *device, struct tc_position *pos) } } -int lin_tape_ibmtape_get_block_in_buffer(void *device, uint32_t *block) +int lin_tape_ibmtape_get_next_block_to_xfer(void *device, struct tc_position *pos) { int rc; char *msg; @@ -2077,12 +2077,13 @@ int lin_tape_ibmtape_get_block_in_buffer(void *device, uint32_t *block) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_READPOS)); memset(&rp, 0, sizeof(struct read_tape_position)); - rp.data_format = RP_SHORT_FORM; + rp.data_format = RP_EXTENDED_FORM; - rc = _sioc_stioc_command(device, STIOC_READ_POSITION_EX, "READPOS SHORT", &rp, &msg); + rc = _sioc_stioc_command(device, STIOC_READ_POSITION_EX, "READPOS EXT", &rp, &msg); if (rc == DEVICE_GOOD) { - *block = (uint32_t)ltfs_betou32(rp.rp_data.rp_short.num_buffer_logical_obj); + pos->partition = rp.rp_data.rp_extended.active_partition; + pos->block = ltfs_betou64(rp.rp_data.rp_extended.last_logical_obj_position); } else { lin_tape_ibmtape_process_errors(device, rc, msg, "get block in buf", true); @@ -2204,17 +2205,19 @@ int lin_tape_ibmtape_format(void *device, TC_FORMAT_TYPE format, const char *vol * @param page page code of log sense * @param buf pointer to buffer to store log data * @param size length of the buffer - * @return 0 on success or a negative value on error + * @return Page length on success or a negative value on error. */ #define MAX_UINT16 (0x0000FFFF) -int lin_tape_ibmtape_logsense_page(void *device, const uint8_t page, const uint8_t subpage, - unsigned char *buf, const size_t size) +int lin_tape_ibmtape_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size) { int rc; char *msg; struct log_sense10_page log_page; + struct lin_tape_ibmtape *priv = (struct lin_tape_ibmtape *) device; + ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOGSENSE)); ltfsmsg(LTFS_DEBUG3, 30597D, "logsense", (unsigned long long)page, (unsigned long long)subpage, ((struct lin_tape_ibmtape *) device)->drive_serial); @@ -2228,23 +2231,14 @@ int lin_tape_ibmtape_logsense_page(void *device, const uint8_t page, const uint8 if (rc != DEVICE_GOOD) { lin_tape_ibmtape_process_errors(device, rc, msg, "logsense page", true); - } - else { + return rc; + } else { memcpy(buf, log_page.data, size); } - return rc; -} - -int lin_tape_ibmtape_logsense(void *device, const uint8_t page, unsigned char *buf, const size_t size) -{ - int ret = 0; - struct lin_tape_ibmtape *priv = (struct lin_tape_ibmtape *) device; - - ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOGSENSE)); - ret = lin_tape_ibmtape_logsense_page(device, page, 0, buf, size); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); - return ret; + + return log_page.len; } /** @@ -2269,8 +2263,8 @@ int lin_tape_ibmtape_remaining_capacity(void *device, struct tc_remaining_cap *c ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_REMAINCAP)); if (IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x05)) { /* Issue LogPage 0x31 */ - rc = lin_tape_ibmtape_logsense(device, LOG_TAPECAPACITY, logdata, LOGSENSEPAGE); - if (rc) { + rc = lin_tape_ibmtape_logsense(device, LOG_TAPECAPACITY, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) { ltfsmsg(LTFS_INFO, 30457I, LOG_TAPECAPACITY, rc); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_REMAINCAP)); return rc; @@ -2309,8 +2303,8 @@ int lin_tape_ibmtape_remaining_capacity(void *device, struct tc_remaining_cap *c } else { /* Issue LogPage 0x17 */ - rc = lin_tape_ibmtape_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); - if (rc) { + rc = lin_tape_ibmtape_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) { ltfsmsg(LTFS_INFO, 30457I, LOG_VOLUMESTATS, rc); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_REMAINCAP)); return rc; @@ -2557,7 +2551,11 @@ int lin_tape_ibmtape_read_attribute(void *device, const tape_partition_t part, c memset(sense, 0, sizeof(sense)); /* Prepare Data Buffer */ - spt.buffer_length = size + 4; + if (size == MAXMAM_SIZE) + spt.buffer_length = MAXMAM_SIZE; + else + spt.buffer_length = size + 4; + spt.buffer = calloc(1, spt.buffer_length); if (spt.buffer == NULL) { ltfsmsg(LTFS_ERR, 10001E, "lin_tape_ibmtape_read_attribute: data buffer"); @@ -2602,7 +2600,13 @@ int lin_tape_ibmtape_read_attribute(void *device, const tape_partition_t part, c id != TC_MAM_APP_FORMAT_VERSION) ltfsmsg(LTFS_INFO, 30460I, rc); } else { - memcpy(buf, (spt.buffer + 4), size); + if (size == MAXMAM_SIZE) { + /* Include header if request size is MAXMAM_SIZE */ + memcpy(buf, spt.buffer, size); + } else { + memcpy(buf, (spt.buffer + 4), size); + } + free(spt.buffer); } @@ -2709,6 +2713,24 @@ int lin_tape_ibmtape_allow_overwrite(void *device, const struct tc_position pos) return rc; } +/** + * GRAO command is currently unsupported on this device + */ +int lin_tape_ibmtape_grao(void *device, unsigned char *buf, const uint32_t len) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + +/** + * RRAO command is currently unsupported on this device + */ +int lin_tape_ibmtape_rrao(void *device, unsigned char *buf, const uint32_t len, size_t *out_size) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + /** * Set compression setting * @param device a pointer to the ibmtape backend @@ -2904,8 +2926,8 @@ int lin_tape_ibmtape_get_cartridge_health(void *device, struct tc_cartridge_heal /* Issue LogPage 0x37 */ cart_health->tape_efficiency = UNSUPPORTED_CARTRIDGE_HEALTH; - rc = lin_tape_ibmtape_logsense(device, LOG_PERFORMANCE, logdata, LOGSENSEPAGE); - if (rc) + rc = lin_tape_ibmtape_logsense(device, LOG_PERFORMANCE, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 30461I, LOG_PERFORMANCE, rc, "get cart health"); else { for(i = 0; i < (int)((sizeof(perfstats)/sizeof(perfstats[0]))); i++) { /* BEAM: loop doesn't iterate - Use loop for future enhancement. */ @@ -2955,8 +2977,9 @@ int lin_tape_ibmtape_get_cartridge_health(void *device, struct tc_cartridge_heal cart_health->read_mbytes = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_begin = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_middle = UNSUPPORTED_CARTRIDGE_HEALTH; - rc = lin_tape_ibmtape_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); - if (rc) + + rc = lin_tape_ibmtape_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 30461I, LOG_VOLUMESTATS, rc, "get cart health"); else { for(i = 0; i < (int)((sizeof(volstats)/sizeof(volstats[0]))); i++) { @@ -3052,8 +3075,8 @@ int lin_tape_ibmtape_get_tape_alert(void *device, uint64_t *tape_alert) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_GETTAPEALT)); /* Issue LogPage 0x2E */ ta = 0; - rc = lin_tape_ibmtape_logsense(device, LOG_TAPE_ALERT, logdata, LOGSENSEPAGE); - if (rc) + rc = lin_tape_ibmtape_logsense(device, LOG_TAPE_ALERT, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 30461I, LOG_TAPE_ALERT, rc, "get tape alert"); else { for(i = 1; i <= 64; i++) { @@ -3393,8 +3416,8 @@ int lin_tape_ibmtape_get_eod_status(void *device, int part) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_GETEODSTAT)); /* Issue LogPage 0x17 */ - rc = lin_tape_ibmtape_logsense(device, LOG_VOL_STATISTICS, logdata, LOGSENSEPAGE); - if (rc) { + rc = lin_tape_ibmtape_logsense(device, LOG_VOL_STATISTICS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (rc < 0) { ltfsmsg(LTFS_WARN, 30464W, LOG_VOL_STATISTICS, rc); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_GETEODSTAT)); return EOD_UNKNOWN; @@ -3462,9 +3485,9 @@ int lin_tape_ibmtape_get_xattr(void *device, const char *name, char **buf) get_current_timespec(&now); if ( priv->fetch_sec_acq_loss_w == 0 || ((priv->fetch_sec_acq_loss_w + 60 < now.tv_sec) && priv->dirty_acq_loss_w)) { - rc = lin_tape_ibmtape_logsense_page(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, - logdata, LOGSENSEPAGE); - if (rc) + rc = lin_tape_ibmtape_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, + logdata, LOGSENSEPAGE); + if (rc < 0) ltfsmsg(LTFS_INFO, 30461I, LOG_PERFORMANCE, rc, "get xattr"); else { if (parse_logPage(logdata, PERF_ACTIVE_CQ_LOSS_W, ¶m_size, logbuf, 16)) { @@ -4374,6 +4397,8 @@ struct tape_ops lin_tape_ibmtape_drive_handler = { .write_attribute = lin_tape_ibmtape_write_attribute, .read_attribute = lin_tape_ibmtape_read_attribute, .allow_overwrite = lin_tape_ibmtape_allow_overwrite, + .grao = lin_tape_ibmtape_grao, + .rrao = lin_tape_ibmtape_rrao, // May be command combination .set_compression = lin_tape_ibmtape_set_compression, .set_default = lin_tape_ibmtape_set_default, @@ -4396,7 +4421,7 @@ struct tape_ops lin_tape_ibmtape_drive_handler = { .get_serialnumber = lin_tape_ibmtape_get_serialnumber, .get_info = lin_tape_ibmtape_get_info, .set_profiler = lin_tape_ibmtape_set_profiler, - .get_block_in_buffer = lin_tape_ibmtape_get_block_in_buffer, + .get_next_block_to_xfer = lin_tape_ibmtape_get_next_block_to_xfer, .is_readonly = lin_tape_ibmtape_is_readonly, }; diff --git a/src/tape_drivers/linux/sg/.gitignore b/src/tape_drivers/linux/sg/.gitignore index ce4d0b0c..0907b2ce 100644 --- a/src/tape_drivers/linux/sg/.gitignore +++ b/src/tape_drivers/linux/sg/.gitignore @@ -1,4 +1,5 @@ vendor_compat.c ibm_tape.c hp_tape.c +quantum_tape.c open_factor.c diff --git a/src/tape_drivers/linux/sg/Makefile.am b/src/tape_drivers/linux/sg/Makefile.am index 1e5948e4..6cc2fa07 100644 --- a/src/tape_drivers/linux/sg/Makefile.am +++ b/src/tape_drivers/linux/sg/Makefile.am @@ -41,7 +41,7 @@ AM_LIBTOOLFLAGS = --tag=disable-static libtape_sg_la_SOURCES = sg_scsi_tape.c sg_tape.c vendor_compat.c ibm_tape.c hp_tape.c quantum_tape.c open_factor.c libtape_sg_la_DEPENDENCIES = ../../../../messages/libtape_linux_sg_dat.a ../../../libltfs/libltfs.la ./libtape_sg_la-reed_solomon_crc.lo ./libtape_sg_la-crc32c_crc.lo libtape_sg_la_LIBADD = ../../../libltfs/libltfs.la ./libtape_sg_la-reed_solomon_crc.lo ./libtape_sg_la-crc32c_crc.lo -libtape_sg_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../../../messages -ltape_linux_sg_dat +libtape_sg_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../../../messages/libtape_linux_sg_dat.a libtape_sg_la_CPPFLAGS = @AM_CPPFLAGS@ @AM_EXTRA_CPPFLAGS@ -I ../../.. -I ../.. vendor_compat.c: diff --git a/src/tape_drivers/linux/sg/sg_tape.c b/src/tape_drivers/linux/sg/sg_tape.c index 8b2a958b..86d93e95 100644 --- a/src/tape_drivers/linux/sg/sg_tape.c +++ b/src/tape_drivers/linux/sg/sg_tape.c @@ -103,11 +103,13 @@ struct sg_global_data global_data; #define TU_DEFAULT_TIMEOUT (60) #define MAX_RETRY (100) +#define MAX_TAKE_DUMP_ATTEMPTS (10) + /* Forward references (For keep function order to struct tape_ops) */ int sg_readpos(void *device, struct tc_position *pos); int sg_locate(void *device, struct tc_position dest, struct tc_position *pos); int sg_space(void *device, size_t count, TC_SPACE_TYPE type, struct tc_position *pos); -int sg_logsense(void *device, const unsigned char page, unsigned char *buf, const size_t size); +int sg_logsense(void *device, const uint8_t page, const uint8_t subpage, unsigned char *buf, const size_t size); int sg_modesense(void *device, const unsigned char page, const TC_MP_PC_TYPE pc, const unsigned char subpage, unsigned char *buf, const size_t size); int sg_modeselect(void *device, unsigned char *buf, const size_t size); @@ -118,19 +120,17 @@ static inline int _parse_logPage(const unsigned char *logdata, const uint16_t param, uint32_t *param_size, unsigned char *buf, const size_t bufsize) { - uint16_t page_len, param_code, param_len; - uint32_t i; + const uint16_t page_len = ((uint16_t)logdata[2] << 8) | (uint16_t)logdata[3]; + uint16_t param_code, param_len; + uint32_t i = LOG_PAGE_HEADER_SIZE; uint32_t ret = -EDEV_INTERNAL_ERROR; - page_len = ((uint16_t)logdata[2] << 8) + (uint16_t)logdata[3]; - i = LOG_PAGE_HEADER_SIZE; - while(i < page_len) { - param_code = ((uint16_t)logdata[i] << 8) + (uint16_t)logdata[i+1]; + param_code = ((uint16_t)logdata[i] << 8) | (uint16_t)logdata[i+1]; param_len = (uint16_t)logdata[i + LOG_PAGE_PARAMSIZE_OFFSET]; - if(param_code == param) + if (param_code == param) { *param_size = param_len; if(bufsize < param_len){ @@ -143,6 +143,7 @@ static inline int _parse_logPage(const unsigned char *logdata, break; } } + i += param_len + LOG_PAGE_PARAM_OFFSET; } @@ -392,6 +393,13 @@ static int _take_dump(struct sg_data *priv, bool capture_unforced) time_t now; struct tm *tm_now; + /* To check if the function became recursive */ + if (priv->recursive_counter > MAX_TAKE_DUMP_ATTEMPTS) { + ltfsmsg(LTFS_WARN, 30297W, priv->recursive_counter); + return 0; + } + priv->recursive_counter++; + if (priv->vendor != VENDOR_IBM) return 0; @@ -426,6 +434,8 @@ static int _take_dump(struct sg_data *priv, bool capture_unforced) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_TAKEDUMPDRV)); + priv->recursive_counter = 0; + return 0; } @@ -487,12 +497,12 @@ static int _raw_open(struct sg_data *priv) priv->dev.fd = -1; return ret; } - priv->vendor = get_vendor_id(id_data.vendor_id); - struct supported_device **cur = get_supported_devs(priv->vendor); + struct supported_device **cur = get_supported_devs(id_data.vendor_id); while(*cur) { if((! strncmp(id_data.vendor_id, (*cur)->vendor_id, strlen((*cur)->vendor_id)) ) && (! strncmp(id_data.product_id, (*cur)->product_id, strlen((*cur)->product_id)) ) ) { + priv->vendor = (*cur)->vendor_type; drive_type = (*cur)->drive_type; break; } @@ -540,12 +550,12 @@ static int _raw_open(struct sg_data *priv) ltfsmsg(LTFS_INFO, 30214I, id_data.product_rev); ltfsmsg(LTFS_INFO, 30215I, priv->drive_serial); - snprintf(priv->info.name, TAPE_DEVNAME_LEN_MAX + 1, "%s", priv->devname); - snprintf(priv->info.vendor, TAPE_VENDOR_NAME_LEN_MAX + 1, "%s", id_data.vendor_id); - snprintf(priv->info.model, TAPE_MODEL_NAME_LEN_MAX + 1, "%s", id_data.product_id); - snprintf(priv->info.serial_number, TAPE_SERIAL_LEN_MAX + 1, "%s", id_data.unit_serial); - snprintf(priv->info.product_rev, PRODUCT_REV_LENGTH + 1, "%s", id_data.product_rev); - snprintf(priv->info.product_name, PRODUCT_NAME_LENGTH + 1, "%s", _generate_product_name(id_data.product_id)); + strncpy(priv->info.name, priv->devname, TAPE_DEVNAME_LEN_MAX + 1); + strncpy(priv->info.vendor, id_data.vendor_id, TAPE_VENDOR_NAME_LEN_MAX + 1); + strncpy(priv->info.model, id_data.product_id, TAPE_MODEL_NAME_LEN_MAX + 1); + strncpy(priv->info.serial_number, id_data.unit_serial, TAPE_SERIAL_LEN_MAX + 1); + strncpy(priv->info.product_rev, id_data.product_rev, PRODUCT_REV_LENGTH + 1); + strncpy(priv->info.product_name, _generate_product_name(id_data.product_id), PRODUCT_NAME_LENGTH + 1); return 0; } @@ -616,6 +626,39 @@ void _clear_por_raw(const int fd) } } +#define _get_stable_tur_response(p) _get_stable_tur_response_raw((p)->dev.fd) + +int _get_stable_tur_response_raw(const int fd) +{ + int i = 0, ret = -1, ret_tur = -1; + + do { + ret_tur = _raw_tur(fd); + if (i == 0) { + /* Keep first return code if it is not an unit attention */ + if (!IS_UNIT_ATTENTION(-ret_tur)) { + ret = ret_tur; + i++; + } + } else if (ret_tur == ret) { + /* Increment counter because it is same as previous response */ + i++; + } else { + /* TUR response is not stable, start over */ + ltfsmsg(LTFS_INFO, 30295I, ret_tur, ret); + if (IS_UNIT_ATTENTION(-ret_tur)) { + ret = -1; + i = 0; + } else { + ret = ret_tur; + i = 1; + } + } + } while (i < 3); + + return ret; +} + /* Forward reference */ int sg_get_device_list(struct tc_drive_info *buf, int count); int sg_reserve(void *device); @@ -770,7 +813,13 @@ static int _reconnect_device(void *device) /* Issue TUR and check reservation conflict happens or not */ _clear_por(priv); - ret = _raw_tur(priv->dev.fd); + + /* + * !!!!! This is a kind of work around to avoid to fetch false one-shot `good` here. + * Fetch result of TUR until 3 straight same result + */ + ltfsmsg(LTFS_INFO, 30296I, __LINE__); + ret = _get_stable_tur_response(priv); if (ret == -EDEV_RESERVATION_CONFLICT) { /* Select another path, recover reservation */ ltfsmsg(LTFS_INFO, 30269I, priv->drive_serial); @@ -785,23 +834,43 @@ static int _reconnect_device(void *device) } else { /* Read reservation information and print */ _clear_por(priv); - memset(&r_info, 0x00, sizeof(r_info)); - f_ret = _fetch_reservation_key(device, &r_info); - if (f_ret == -EDEV_NO_RESERVATION_HOLDER) { - /* Real POR may happens */ - ltfsmsg(LTFS_INFO, 30270I, priv->drive_serial); + + /* + * !!!!! This is the code just in case, check TUR response again and restore reservation + * if drive reports `reservation conflict`. + */ + ltfsmsg(LTFS_INFO, 30296I, __LINE__); + ret = _get_stable_tur_response(priv); + if (ret == -EDEV_RESERVATION_CONFLICT) { + /* Select another path, recover reservation */ + ltfsmsg(LTFS_INFO, 30269I, priv->drive_serial); _register_key(priv, priv->key); - ret = sg_reserve(device); + ret = _cdb_pro(device, PRO_ACT_PREEMPT_ABORT, PRO_TYPE_EXCLUSIVE, + priv->key, priv->key); if (!ret) { ltfsmsg(LTFS_INFO, 30272I, priv->drive_serial); _clear_por(priv); - ret = -EDEV_REAL_POWER_ON_RESET; + ret = -EDEV_NEED_FAILOVER; } } else { - /* Select same path */ - ltfsmsg(LTFS_INFO, 30271I, priv->drive_serial); - _clear_por(priv); - ret = -EDEV_NEED_FAILOVER; + memset(&r_info, 0x00, sizeof(r_info)); + f_ret = _fetch_reservation_key(device, &r_info); + if (f_ret == -EDEV_NO_RESERVATION_HOLDER) { + /* Real POR may happens */ + ltfsmsg(LTFS_INFO, 30270I, priv->drive_serial); + _register_key(priv, priv->key); + ret = sg_reserve(device); + if (!ret) { + ltfsmsg(LTFS_INFO, 30272I, priv->drive_serial); + _clear_por(priv); + ret = -EDEV_REAL_POWER_ON_RESET; + } + } else { + /* Select same path */ + ltfsmsg(LTFS_INFO, 30271I, priv->drive_serial); + _clear_por(priv); + ret = -EDEV_NEED_FAILOVER; + } } } @@ -1169,6 +1238,57 @@ static int _register_key(void *device, unsigned char *key) return ret; } +/** SCSI command handling of REPORT SUPPORTED OPERATION CODES + */ +static int _cdb_rsoc(void* device, unsigned char *buf, uint32_t len) +{ + int ret = -EDEV_UNKNOWN; + int ret_ep = DEVICE_GOOD; + struct sg_data *priv = (struct sg_data*)device; + + sg_io_hdr_t req; + unsigned char cdb[CDB12_LEN]; + unsigned char sense[MAXSENSE]; + int timeout = 60; + char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "RSOC"; + char *msg = NULL; + + /* Zero out the CDB and the result buffer */ + ret = init_sg_io_header(&req); + if (ret < 0) + return ret; + + memset(cdb, 0, sizeof(cdb)); + memset(buf, 0, len); + memset(sense, 0, sizeof(sense)); + + /* Build CDB */ + cdb[0] = MAINTENANCE_IN; + cdb[1] = 0x0C; /* REPORT SUPPORTED OPERATION CODES */ + cdb[2] = 0x80; /* Fetch all commands with RCTD */ + ltfs_u32tobe(cdb + 6, len); /* allocation length */ + + /* Build request */ + req.dxfer_direction = SCSI_FROM_TARGET_TO_INITIATOR; + req.cmd_len = sizeof(cdb); + req.mx_sb_len = sizeof(sense); + req.dxfer_len = len; + req.dxferp = buf; + req.cmdp = cdb; + req.sbp = sense; + req.timeout = SGConversion(timeout); + req.usr_ptr = (void *)cmd_desc; + + ret = sg_issue_cdb_command(&priv->dev, &req, &msg); + if (ret < 0){ + ret_ep = _process_errors(device, ret, msg, cmd_desc, true, true); + if (ret_ep < 0) + ret = ret_ep; + } + + return ret; +} + /* Global functions */ int sg_open(const char *devname, void **handle) { @@ -1180,6 +1300,9 @@ int sg_open(const char *devname, void **handle) struct sg_data *priv; + unsigned char *rsoc_buf = NULL; + uint32_t rsoc_len = RSOC_BUF_SIZE; + CHECK_ARG_NULL(devname, -LTFS_NULL_ARG); CHECK_ARG_NULL(handle, -LTFS_NULL_ARG); @@ -1268,7 +1391,7 @@ int sg_open(const char *devname, void **handle) ret = DEVICE_GOOD; break; } else if (ret < 0) { - ltfsmsg(LTFS_INFO, 30289I, priv->devname); + ltfsmsg(LTFS_INFO, 30289I, priv->devname, ret); close(priv->dev.fd); priv->dev.fd = -1; free(priv->devname); @@ -1316,18 +1439,53 @@ int sg_open(const char *devname, void **handle) ioctl(priv->dev.fd, SG_SET_RESERVED_SIZE, &reserved_size); ret = ioctl(priv->dev.fd, SG_GET_RESERVED_SIZE, &reserved_size); if (ret < 0) { - ltfsmsg(LTFS_ERR, 30284E, devname); + ltfsmsg(LTFS_INFO, 30284I, devname); goto free; } ltfsmsg(LTFS_INFO, 30285I, devname, reserved_size); increment_openfactor(priv->info.host, priv->info.channel); - /* Setup vendor specific parameters */ + /* Setup error table sense to ltfs error code */ init_error_table(priv->vendor, &standard_table, &vendor_table); - init_timeout(priv->vendor, &priv->timeouts, priv->drive_type); - if (!priv->timeouts) - ibm_tape_init_timeout(&priv->timeouts, priv->drive_type); + + /* Setup device specific timeout value */ + rsoc_buf = calloc(1, RSOC_BUF_SIZE); + if (rsoc_buf) { + ret = _cdb_rsoc(&priv->dev, rsoc_buf, RSOC_BUF_SIZE); + rsoc_len = ltfs_betou32(rsoc_buf); + if (!ret && rsoc_len < RSOC_BUF_SIZE) { + ltfsmsg(LTFS_INFO, 30294I, "RSOC"); + ret = init_timeout_rsoc(&priv->timeouts, rsoc_buf, rsoc_len); + if (!priv->timeouts) + ibm_tape_init_timeout(&priv->timeouts, priv->drive_type); + } + + if (ret < 0) { + /* + * The drive doesn't support RSOC, buffer overrun or parse error + * try to initialize the timeout table from drive vendor and drive type + */ + ltfsmsg(LTFS_INFO, 30294I, "vendor and device"); + ret = init_timeout(priv->vendor, &priv->timeouts, priv->drive_type); + if (!priv->timeouts) { + ltfsmsg(LTFS_INFO, 30294I, "device"); + ibm_tape_init_timeout(&priv->timeouts, priv->drive_type); + } + } + free(rsoc_buf); + } else { + /* + * Memory allocation failure, try to initialize the timeout table + * from drive vendor and drive type + */ + ltfsmsg(LTFS_INFO, 30294I, "vendor and device"); + init_timeout(priv->vendor, &priv->timeouts, priv->drive_type); + if (!priv->timeouts) { + ltfsmsg(LTFS_INFO, 30294I, "device"); + ibm_tape_init_timeout(&priv->timeouts, priv->drive_type); + } + } /* Issue TURs to clear POR sense */ _clear_por(priv); @@ -2595,11 +2753,17 @@ int sg_load(void *device, struct tc_position *pos) priv->density_code = buf[8]; - if (priv->vendor == VENDOR_HP) { + if (buf[2] == 0x00 || buf[2] == 0x01) { + /* + * Non-IBM drive doesn't have cartridge type so need to assume from density code. + */ priv->cart_type = assume_cart_type(priv->density_code); if (buf[2] == 0x01) priv->is_worm = true; } else { + /* + * IBM drive haves cartridge type in buf[2] like TC_MP_LTO5D_CART. + */ priv->cart_type = buf[2]; } @@ -2660,7 +2824,7 @@ int sg_readpos(void *device, struct tc_position *pos) struct sg_data *priv = (struct sg_data*)device; sg_io_hdr_t req; - unsigned char cdb[CDB6_LEN]; + unsigned char cdb[CDB10_LEN]; unsigned char sense[MAXSENSE]; int timeout; char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "READPOS"; @@ -2873,9 +3037,12 @@ int sg_remaining_capacity(void *device, struct tc_remaining_cap *cap) memset(buffer, 0, LOGSENSEPAGE); - if (IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x05)) { + if ((IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x05)) || + (priv->vendor == VENDOR_HP && IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x06)) || + (priv->vendor == VENDOR_QUANTUM_B && IS_LTO(priv->drive_type))) { + /* Use LogPage 0x31 */ - ret = sg_logsense(device, (uint8_t)LOG_TAPECAPACITY, (void *)buffer, LOGSENSEPAGE); + ret = sg_logsense(device, (uint8_t)LOG_TAPECAPACITY, (uint8_t)0, (void *)buffer, LOGSENSEPAGE); if(ret < 0) { ltfsmsg(LTFS_INFO, 30229I, LOG_VOLUMESTATS, ret); @@ -2930,7 +3097,7 @@ int sg_remaining_capacity(void *device, struct tc_remaining_cap *cap) ret = DEVICE_GOOD; } else { /* Use LogPage 0x17 */ - ret = sg_logsense(device, LOG_VOLUMESTATS, (void *)buffer, LOGSENSEPAGE); + ret = sg_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, (void *)buffer, LOGSENSEPAGE); if(ret < 0) { ltfsmsg(LTFS_INFO, 30229I, LOG_VOLUMESTATS, ret); @@ -2999,8 +3166,8 @@ int sg_remaining_capacity(void *device, struct tc_remaining_cap *cap) return ret; } -static int _cdb_logsense(void *device, const unsigned char page, const unsigned char subpage, - unsigned char *buf, const size_t size) +int sg_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size) { int ret = -EDEV_UNKNOWN; int ret_ep = DEVICE_GOOD; @@ -3013,12 +3180,23 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "LOGSENSE"; char *msg = NULL; + unsigned int len = 0; + unsigned char *inner_buf = NULL; + ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOGSENSE)); + ltfsmsg(LTFS_DEBUG3, 30397D, "logsense", + (unsigned long long)page, (unsigned long long)subpage, priv->drive_serial); + + inner_buf = calloc(1, MAXLP_SIZE); /* Assume max length of LP is 0xFFFF */ + if (!inner_buf) + return -LTFS_NO_MEMORY; /* Zero out the CDB and the result buffer */ ret = init_sg_io_header(&req); - if (ret < 0) + if (ret < 0) { + free(inner_buf); return ret; + } memset(cdb, 0, sizeof(cdb)); memset(sense, 0, sizeof(sense)); @@ -3027,41 +3205,43 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned cdb[0] = LOG_SENSE; cdb[2] = 0x40 | (page & 0x3F); /* Current value */ cdb[3] = subpage; - ltfs_u16tobe(cdb + 7, size); + ltfs_u16tobe(cdb + 7, MAXLP_SIZE); timeout = get_timeout(priv->timeouts, cdb[0]); - if (timeout < 0) + if (timeout < 0) { + free(inner_buf); return -EDEV_UNSUPPORETD_COMMAND; + } /* Build request */ req.dxfer_direction = SCSI_FROM_TARGET_TO_INITIATOR; req.cmd_len = sizeof(cdb); req.mx_sb_len = sizeof(sense); - req.dxfer_len = size; - req.dxferp = buf; + req.dxfer_len = MAXLP_SIZE; + req.dxferp = inner_buf; req.cmdp = cdb; req.sbp = sense; req.timeout = SGConversion(timeout); req.usr_ptr = (void *)cmd_desc; ret = sg_issue_cdb_command(&priv->dev, &req, &msg); - if (ret < 0){ + if (ret < 0) { ret_ep = _process_errors(device, ret, msg, cmd_desc, true, true); if (ret_ep < 0) ret = ret_ep; - } - - ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); + } else { + len = ((int)inner_buf[2] << 8) + (int)inner_buf[3] + 4; - return ret; -} + if (size > len) + memcpy(buf, inner_buf, len); + else + memcpy(buf, inner_buf, size); -int sg_logsense(void *device, const unsigned char page, unsigned char *buf, const size_t size) -{ - int ret = -EDEV_UNKNOWN; + ret = len; + } - ltfsmsg(LTFS_DEBUG3, 30393D, "logsense", page, ""); - ret = _cdb_logsense(device, page, 0x00, buf, size); + free (inner_buf); + ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); return ret; } @@ -3409,7 +3589,12 @@ int sg_read_attribute(void *device, const tape_partition_t part, ltfsmsg(LTFS_DEBUG3, 30397D, "readattr", (unsigned long long)part, (unsigned long long)id, priv->drive_serial); /* Prepare the buffer to transfer */ - uint32_t len = size + 4; + uint32_t len = 0; + if (size == MAXMAM_SIZE) + len = MAXMAM_SIZE; + else + len = size + 4; + unsigned char *buffer = calloc(1, len); if (!buffer) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); @@ -3467,7 +3652,12 @@ int sg_read_attribute(void *device, const tape_partition_t part, id != TC_MAM_APP_FORMAT_VERSION) ltfsmsg(LTFS_INFO, 30233I, ret); } else { - memcpy(buf, buffer + 4, size); + if (size == MAXMAM_SIZE) { + /* Include header if request size is MAXMAM_SIZE */ + memcpy(buf, buffer, size); + } else { + memcpy(buf, buffer + 4, size); + } } free(buffer); @@ -3666,8 +3856,8 @@ int sg_get_cartridge_health(void *device, struct tc_cartridge_health *cart_healt /* Issue LogPage 0x37 */ cart_health->tape_efficiency = UNSUPPORTED_CARTRIDGE_HEALTH; - ret = sg_logsense(device, LOG_PERFORMANCE, logdata, LOGSENSEPAGE); - if (ret) + ret = sg_logsense(device, LOG_PERFORMANCE, (uint8_t)0, logdata, LOGSENSEPAGE); + if (ret < 0) ltfsmsg(LTFS_INFO, 30234I, LOG_PERFORMANCE, ret, "get cart health"); else { for(i = 0; i < (int)((sizeof(perfstats)/sizeof(perfstats[0]))); i++) { @@ -3721,7 +3911,7 @@ int sg_get_cartridge_health(void *device, struct tc_cartridge_health *cart_healt cart_health->read_mbytes = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_begin = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_middle = UNSUPPORTED_CARTRIDGE_HEALTH; - ret = sg_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); + ret = sg_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); if (ret < 0) ltfsmsg(LTFS_INFO, 30234I, LOG_VOLUMESTATS, ret, "get cart health"); else { @@ -3817,10 +4007,11 @@ int sg_get_tape_alert(void *device, uint64_t *tape_alert) /* Issue LogPage 0x2E */ ta = 0; - ret = sg_logsense(device, LOG_TAPE_ALERT, logdata, LOGSENSEPAGE); + ret = sg_logsense(device, LOG_TAPE_ALERT, (uint8_t)0, logdata, LOGSENSEPAGE); if (ret < 0) ltfsmsg(LTFS_INFO, 30234I, LOG_TAPE_ALERT, ret, "get tape alert"); else { + ret = 0; for(i = 1; i <= 64; i++) { if (_parse_logPage(logdata, (uint16_t) i, ¶m_size, buf, 16) || param_size != sizeof(uint8_t)) { @@ -3873,11 +4064,11 @@ int sg_get_xattr(void *device, const char *name, char **buf) if (priv->fetch_sec_acq_loss_w == 0 || ((priv->fetch_sec_acq_loss_w + 60 < now.tv_sec) && priv->dirty_acq_loss_w)) { - ret = _cdb_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, logdata, LOGSENSEPAGE); - + ret = sg_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, logdata, LOGSENSEPAGE); if (ret < 0) { ltfsmsg(LTFS_INFO, 30234I, LOG_PERFORMANCE, ret, "get xattr"); } else { + ret = 0; if (_parse_logPage(logdata, PERF_ACTIVE_CQ_LOSS_W, ¶m_size, logbuf, 16)) { ltfsmsg(LTFS_INFO, 30235I, LOG_PERFORMANCE, "get xattr"); ret = -LTFS_NO_XATTR; @@ -4128,8 +4319,8 @@ int sg_get_eod_status(void *device, int part) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_GETEODSTAT)); /* Issue LogPage 0x17 */ - ret = sg_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); - if (ret) { + ret = sg_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (ret < 0) { ltfsmsg(LTFS_WARN, 30237W, LOG_VOLUMESTATS, ret); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_GETEODSTAT)); return EOD_UNKNOWN; @@ -4250,12 +4441,12 @@ int sg_get_device_list(struct tc_drive_info *buf, int count) } if (found < count && buf) { - snprintf(buf[found].name, TAPE_DEVNAME_LEN_MAX + 1, "%s", devname); - snprintf(buf[found].vendor, TAPE_VENDOR_NAME_LEN_MAX + 1, "%s", identifier.vendor_id); - snprintf(buf[found].model, TAPE_MODEL_NAME_LEN_MAX + 1, "%s", identifier.product_id); - snprintf(buf[found].serial_number, TAPE_SERIAL_LEN_MAX + 1, "%s", identifier.unit_serial); - snprintf(buf[found].product_rev, PRODUCT_REV_LENGTH + 1, "%s", identifier.product_rev); - snprintf(buf[found].product_name, PRODUCT_NAME_LENGTH + 1, "%s", _generate_product_name(identifier.product_id)); + strncpy(buf[found].name, devname, TAPE_DEVNAME_LEN_MAX + 1); + strncpy(buf[found].vendor, identifier.vendor_id, TAPE_VENDOR_NAME_LEN_MAX + 1); + strncpy(buf[found].model, identifier.product_id, TAPE_MODEL_NAME_LEN_MAX + 1); + strncpy(buf[found].serial_number, identifier.unit_serial, TAPE_SERIAL_LEN_MAX + 1); + strncpy(buf[found].product_rev, identifier.product_rev, PRODUCT_REV_LENGTH + 1); + strncpy(buf[found].product_name, _generate_product_name(identifier.product_id), PRODUCT_NAME_LENGTH + 1); if (! ioctl(dev.fd, SG_GET_SCSI_ID, &scsi_id)) { buf[found].host = scsi_id.host_no; @@ -4843,14 +5034,14 @@ int sg_set_profiler(void *device, char *work_dir, bool enable) return rc; } -int sg_get_block_in_buffer(void *device, uint32_t *block) +int sg_get_next_block_to_xfer(void *device, struct tc_position *pos) { int ret = -EDEV_UNKNOWN; int ret_ep = DEVICE_GOOD; struct sg_data *priv = (struct sg_data*)device; sg_io_hdr_t req; - unsigned char cdb[CDB6_LEN]; + unsigned char cdb[CDB10_LEN]; unsigned char sense[MAXSENSE]; int timeout; char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "READPOS"; @@ -4859,6 +5050,8 @@ int sg_get_block_in_buffer(void *device, uint32_t *block) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_READPOS)); + memset(pos, 0, sizeof(struct tc_position)); + /* Zero out the CDB and the result buffer */ ret = init_sg_io_header(&req); if (ret < 0) @@ -4870,6 +5063,7 @@ int sg_get_block_in_buffer(void *device, uint32_t *block) /* Build CDB */ cdb[0] = READ_POSITION; cdb[1] = 0x08; /* Extended Format */ + ltfs_u16tobe(cdb + 7, sizeof(buf)); /* allocation length */ timeout = get_timeout(priv->timeouts, cdb[0]); if (timeout < 0) @@ -4888,10 +5082,11 @@ int sg_get_block_in_buffer(void *device, uint32_t *block) ret = sg_issue_cdb_command(&priv->dev, &req, &msg); if (ret == DEVICE_GOOD) { - *block = (buf[5] << 16) + (buf[6] << 8) + (int)buf[7]; + pos->partition = (tape_partition_t)buf[1]; + pos->block = ltfs_betou64(buf + 16); - ltfsmsg(LTFS_DEBUG, 30398D, "blocks-in-buffer", - (unsigned long long)*block, (unsigned long long)0, (unsigned long long)0, priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30398D, "next-block-to-xfer", + (unsigned long long)pos->partition, (unsigned long long)pos->block, (unsigned long long)0, priv->drive_serial); } else { ret_ep = _process_errors(device, ret, msg, cmd_desc, true, true); if (ret_ep < 0) @@ -4903,6 +5098,135 @@ int sg_get_block_in_buffer(void *device, uint32_t *block) return ret; } +int sg_grao(void *device, unsigned char *buf, const uint32_t len) +{ + int ret = -EDEV_UNKNOWN; + int ret_ep = DEVICE_GOOD; + + struct sg_data *priv = (struct sg_data*)device; + sg_io_hdr_t req; + unsigned char cdb[CDB12_LEN]; + unsigned char sense[MAXSENSE]; + int timeout; + char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "GRAO"; + char *msg = NULL; + + if (IS_LTO(priv->drive_type)) { + if ( DRIVE_GEN(priv->drive_type) == 0x05 || DRIVE_GEN(priv->drive_type) == 0x06 || + DRIVE_GEN(priv->drive_type) == 0x07 || DRIVE_GEN(priv->drive_type) == 0x08 || + DRIVE_GEN(priv->drive_type) == DRIVE_LTO9_HH ) { + /* LTO5-LTO8 and LTO9-HH don't support RAO commands */ + return -EDEV_UNSUPPORETD_COMMAND; + } + } + + /* Zero out the CDB and the result buffer */ + ret = init_sg_io_header(&req); + if (ret < 0) + return ret; + + memset(cdb, 0, sizeof(cdb)); + memset(sense, 0, sizeof(sense)); + + /* Build CDB */ + cdb[0] = MAINTENANCE_OUT; /* op_code */ + cdb[1] = 0x1D; /* service action */ + cdb[2] = 0x2; /* PROCESS, reorder UDS on */ + cdb[3] = LTFS_GEOMETORY_OFF; /* UDS_TYPE */ + ltfs_u32tobe(cdb + 6, len); /* parameter list len starts from 6 byte, adding len to cbc 6-9 */ + + timeout = get_timeout(priv->timeouts, cdb[0]); + if (timeout < 0) + return -EDEV_UNSUPPORETD_COMMAND; + + /* Build request */ + req.dxfer_direction = SCSI_FROM_INITIATOR_TO_TARGET; + req.cmd_len = sizeof(cdb); + req.mx_sb_len = sizeof(sense); + req.dxfer_len = len; + req.dxferp = buf; + req.cmdp = cdb; + req.sbp = sense; + req.timeout = SGConversion(timeout); + req.usr_ptr = (void *)cmd_desc; + + ret = sg_issue_cdb_command(&priv->dev, &req, &msg); + if (ret < 0){ + ret_ep = _process_errors(device, ret, msg, cmd_desc, true, true); + if (ret_ep < 0) + ret = ret_ep; + } + + return ret; +} + +int sg_rrao(void *device, unsigned char *buf, const uint32_t len, size_t *out_size) +{ + int ret = -EDEV_UNKNOWN; + int ret_ep = DEVICE_GOOD; + + struct sg_data *priv = (struct sg_data*)device; + sg_io_hdr_t req; + unsigned char sense[MAXSENSE]; + unsigned char cdb[CDB12_LEN]; + int timeout; + char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "RRAO"; + char *msg = NULL; + + *out_size = 0; + + if (IS_LTO(priv->drive_type)) { + if ( DRIVE_GEN(priv->drive_type) == 0x05 || DRIVE_GEN(priv->drive_type) == 0x06 || + DRIVE_GEN(priv->drive_type) == 0x07 || DRIVE_GEN(priv->drive_type) == 0x08 ) { + /* LTO6-LTO8 don't support RAO commands */ + return -EDEV_UNSUPPORETD_COMMAND; + } + } + + /* Zero out the CDB and the result buffer */ + ret = init_sg_io_header(&req); + if (ret < 0) + return ret; + memset(cdb, 0, sizeof(cdb)); + memset(sense, 0, sizeof(sense)); + + /* Build CDB */ + cdb[0] = MAINTENANCE_IN; + cdb[1] = 0x1D; /* Do not set UDS_LIMITS (0x80) */ + ltfs_u32tobe(cdb + 6, len); + cdb[10] = LTFS_GEOMETORY_OFF; /* UDS_TYPE */ + + timeout = get_timeout(priv->timeouts, cdb[0]); + if (timeout < 0) { + return -EDEV_UNSUPPORETD_COMMAND; + } + + /* Build request */ + req.dxfer_direction = SCSI_FROM_TARGET_TO_INITIATOR; + req.cmd_len = sizeof(cdb); + req.mx_sb_len = sizeof(sense); + req.dxfer_len = len; + req.dxferp = buf; + req.cmdp = cdb; + req.sbp = sense; + req.timeout = SGConversion(timeout); + req.usr_ptr = (void *)cmd_desc; + + ret = sg_issue_cdb_command(&priv->dev, &req, &msg); + if (ret < 0) { + ret_ep = _process_errors(device, ret, msg, cmd_desc, true, true); + if (ret_ep < 0) + ret = ret_ep; + } + + if (!ret) { + /* Set real length returned from the drive */ + *out_size = ltfs_betou32(buf + 4) + 8; /* Additional Data length and header */ + } + + return ret; +} + struct tape_ops sg_handler = { .open = sg_open, .reopen = sg_reopen, @@ -4935,6 +5259,8 @@ struct tape_ops sg_handler = { .write_attribute = sg_write_attribute, .read_attribute = sg_read_attribute, .allow_overwrite = sg_allow_overwrite, + .grao = sg_grao, + .rrao = sg_rrao, // May be command combination .set_compression = sg_set_compression, .set_default = sg_set_default, @@ -4957,7 +5283,7 @@ struct tape_ops sg_handler = { .get_serialnumber = sg_get_serialnumber, .get_info = sg_get_info, .set_profiler = sg_set_profiler, - .get_block_in_buffer = sg_get_block_in_buffer, + .get_next_block_to_xfer = sg_get_next_block_to_xfer, .is_readonly = sg_is_readonly, }; diff --git a/src/tape_drivers/linux/sg/sg_tape.h b/src/tape_drivers/linux/sg/sg_tape.h index 4d9de00e..2220d0f1 100644 --- a/src/tape_drivers/linux/sg/sg_tape.h +++ b/src/tape_drivers/linux/sg/sg_tape.h @@ -82,6 +82,7 @@ struct sg_data { struct timeout_tape *timeouts; /**< Timeout table */ struct tc_drive_info info; /**< Drive information */ FILE* profiler; /**< The file pointer for profiler */ + int recursive_counter; /**< Recursive counter for take dump */ }; struct sg_global_data { diff --git a/src/tape_drivers/netbsd/scsipi-ibmtape/Makefile.am b/src/tape_drivers/netbsd/scsipi-ibmtape/Makefile.am index 64ac236a..1515a5ed 100644 --- a/src/tape_drivers/netbsd/scsipi-ibmtape/Makefile.am +++ b/src/tape_drivers/netbsd/scsipi-ibmtape/Makefile.am @@ -41,7 +41,7 @@ AM_LIBTOOLFLAGS = --tag=disable-static libtape_scsipi_ibmtape_la_SOURCES = scsipi_scsi_tape.c scsipi_ibmtape.c vendor_compat.c ibm_tape.c hp_tape.c quantum_tape.c libtape_scsipi_ibmtape_la_DEPENDENCIES = ../../../../messages/libtape_linux_sg_ibmtape_dat.a ../../../libltfs/libltfs.la libtape_scsipi_ibmtape_la-reed_solomon_crc.lo libtape_scsipi_ibmtape_la-crc32c_crc.lo libtape_scsipi_ibmtape_la_LIBADD = ../../../libltfs/libltfs.la libtape_scsipi_ibmtape_la-reed_solomon_crc.lo libtape_scsipi_ibmtape_la-crc32c_crc.lo -libtape_scsipi_ibmtape_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../../../messages -ltape_linux_sg_ibmtape_dat +libtape_scsipi_ibmtape_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../../../messages/libtape_linux_sg_ibmtape_dat.a libtape_scsipi_ibmtape_la_CPPFLAGS = @AM_CPPFLAGS@ @AM_EXTRA_CPPFLAGS@ -I ../../.. -I ../.. vendor_compat.c: diff --git a/src/tape_drivers/netbsd/scsipi-ibmtape/scsipi_ibmtape.c b/src/tape_drivers/netbsd/scsipi-ibmtape/scsipi_ibmtape.c index 8dc7399a..0ea8df55 100644 --- a/src/tape_drivers/netbsd/scsipi-ibmtape/scsipi_ibmtape.c +++ b/src/tape_drivers/netbsd/scsipi-ibmtape/scsipi_ibmtape.c @@ -101,7 +101,8 @@ struct scsipi_ibmtape_global_data global_data; int scsipi_ibmtape_readpos(void *device, struct tc_position *pos); int scsipi_ibmtape_locate(void *device, struct tc_position dest, struct tc_position *pos); int scsipi_ibmtape_space(void *device, size_t count, TC_SPACE_TYPE type, struct tc_position *pos); -int scsipi_ibmtape_logsense(void *device, const unsigned char page, unsigned char *buf, const size_t size); +int scsipi_ibmtape_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size); int scsipi_ibmtape_modesense(void *device, const unsigned char page, const TC_MP_PC_TYPE pc, const unsigned char subpage, unsigned char *buf, const size_t size); int scsipi_ibmtape_modeselect(void *device, unsigned char *buf, const size_t size); @@ -2376,7 +2377,7 @@ int scsipi_ibmtape_readpos(void *device, struct tc_position *pos) struct scsipi_ibmtape_data *priv = (struct scsipi_ibmtape_data*)device; scsireq_t req; - unsigned char cdb[CDB6_LEN]; + unsigned char cdb[CDB10_LEN]; int timeout; char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "READPOS"; char *msg = NULL; @@ -2576,7 +2577,7 @@ int scsipi_ibmtape_remaining_capacity(void *device, struct tc_remaining_cap *cap if (IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x05)) { /* Use LogPage 0x31 */ - ret = scsipi_ibmtape_logsense(device, (uint8_t)LOG_TAPECAPACITY, (void *)buffer, LOGSENSEPAGE); + ret = scsipi_ibmtape_logsense(device, (uint8_t)LOG_TAPECAPACITY, (uint8_t)0, (void *)buffer, LOGSENSEPAGE); if(ret < 0) { ltfsmsg(LTFS_INFO, 30229I, LOG_VOLUMESTATS, ret); @@ -2631,7 +2632,7 @@ int scsipi_ibmtape_remaining_capacity(void *device, struct tc_remaining_cap *cap ret = DEVICE_GOOD; } else { /* Use LogPage 0x17 */ - ret = scsipi_ibmtape_logsense(device, LOG_VOLUMESTATS, (void *)buffer, LOGSENSEPAGE); + ret = scsipi_ibmtape_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, (void *)buffer, LOGSENSEPAGE); if(ret < 0) { ltfsmsg(LTFS_INFO, 30229I, LOG_VOLUMESTATS, ret); @@ -2700,8 +2701,8 @@ int scsipi_ibmtape_remaining_capacity(void *device, struct tc_remaining_cap *cap return ret; } -static int _cdb_logsense(void *device, const unsigned char page, const unsigned char subpage, - unsigned char *buf, const size_t size) +int scsipi_ibmtape_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size) { int ret = -EDEV_UNKNOWN; int ret_ep = DEVICE_GOOD; @@ -2713,8 +2714,16 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "LOGSENSE"; char *msg = NULL; + unsigned int len = 0; + unsigned char *inner_buf = NULL; + + ltfsmsg(LTFS_DEBUG3, 30397D, "logsense", page, subpage, priv->drive_serial); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOGSENSE)); + inner_buf = calloc(1, MAXLP_SIZE); /* Assume max length of LP is 0xFFFF */ + if (!inner_buf) + return -LTFS_NO_MEMORY; + /* Zero out the CDB and the result buffer */ ret = init_scsireq(&req); if (ret < 0) @@ -2726,7 +2735,7 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned cdb[0] = LOG_SENSE; cdb[2] = 0x40 | (page & 0x3F); /* Current value */ cdb[3] = subpage; - ltfs_u16tobe(cdb + 7, size); + ltfs_u16tobe(cdb + 7, MAXLP_SIZE); timeout = ibm_tape_get_timeout(priv->timeouts, cdb[0]); if (timeout < 0) @@ -2745,19 +2754,19 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned ret_ep = _process_errors(device, ret, msg, cmd_desc, true, true); if (ret_ep < 0) ret = ret_ep; - } - - ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); + } else { + len = ((int)inner_buf[2] << 8) + (int)inner_buf[3] + 4; - return ret; -} + if (size > len) + memcpy(buf, inner_buf, len); + else + memcpy(buf, inner_buf, size); -int scsipi_ibmtape_logsense(void *device, const unsigned char page, unsigned char *buf, const size_t size) -{ - int ret = -EDEV_UNKNOWN; + ret = len; + } - ltfsmsg(LTFS_DEBUG3, 30393D, "logsense", page, ""); - ret = _cdb_logsense(device, page, 0x00, buf, size); + free (inner_buf); + ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); return ret; } @@ -3081,7 +3090,12 @@ int scsipi_ibmtape_read_attribute(void *device, const tape_partition_t part, ltfsmsg(LTFS_DEBUG3, 30397D, "readattr", (unsigned long long)part, (unsigned long long)id, priv->drive_serial); /* Prepare the buffer to transfer */ - uint32_t len = size + 4; + uint32_t len = 0; + if (size == MAXMAM_SIZE) + len = MAXMAM_SIZE; + else + len = size + 4; + unsigned char *buffer = calloc(1, len); if (!buffer) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); @@ -3135,7 +3149,12 @@ int scsipi_ibmtape_read_attribute(void *device, const tape_partition_t part, id != TC_MAM_APP_FORMAT_VERSION) ltfsmsg(LTFS_INFO, 30233I, ret); } else { - memcpy(buf, buffer + 4, size); + if (size == MAXMAM_SIZE) { + /* Include header if request size is MAXMAM_SIZE */ + memcpy(buf, buffer, size); + } else { + memcpy(buf, buffer + 4, size); + } } free(buffer); @@ -3199,6 +3218,24 @@ int scsipi_ibmtape_allow_overwrite(void *device, const struct tc_position pos) return ret; } +/** + * GRAO command is currently unsupported on this device + */ +int scsipi_ibmtape_grao(void *device, unsigned char *buf, const uint32_t len) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + +/** + * RRAO command is currently unsupported on this device + */ +int scsipi_ibmtape_rrao(void *device, unsigned char *buf, const uint32_t len, size_t *out_size) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + int scsipi_ibmtape_set_compression(void *device, const bool enable_compression, struct tc_position *pos) { int ret = -EDEV_UNKNOWN; @@ -3325,8 +3362,8 @@ int scsipi_ibmtape_get_cartridge_health(void *device, struct tc_cartridge_health /* Issue LogPage 0x37 */ cart_health->tape_efficiency = UNSUPPORTED_CARTRIDGE_HEALTH; - ret = scsipi_ibmtape_logsense(device, LOG_PERFORMANCE, logdata, LOGSENSEPAGE); - if (ret) + ret = scsipi_ibmtape_logsense(device, LOG_PERFORMANCE, (uint8_t)0, logdata, LOGSENSEPAGE); + if (ret < 0) ltfsmsg(LTFS_INFO, 30234I, LOG_PERFORMANCE, ret, "get cart health"); else { for(i = 0; i < (int)((sizeof(perfstats)/sizeof(perfstats[0]))); i++) { @@ -3380,7 +3417,8 @@ int scsipi_ibmtape_get_cartridge_health(void *device, struct tc_cartridge_health cart_health->read_mbytes = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_begin = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_middle = UNSUPPORTED_CARTRIDGE_HEALTH; - ret = scsipi_ibmtape_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); + + ret = scsipi_ibmtape_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); if (ret < 0) ltfsmsg(LTFS_INFO, 30234I, LOG_VOLUMESTATS, ret, "get cart health"); else { @@ -3476,10 +3514,11 @@ int scsipi_ibmtape_get_tape_alert(void *device, uint64_t *tape_alert) /* Issue LogPage 0x2E */ ta = 0; - ret = scsipi_ibmtape_logsense(device, LOG_TAPE_ALERT, logdata, LOGSENSEPAGE); + ret = scsipi_ibmtape_logsense(device, LOG_TAPE_ALERT, (uint8_t)0, logdata, LOGSENSEPAGE); if (ret < 0) ltfsmsg(LTFS_INFO, 30234I, LOG_TAPE_ALERT, ret, "get tape alert"); else { + ret = 0; for(i = 1; i <= 64; i++) { if (_parse_logPage(logdata, (uint16_t) i, ¶m_size, buf, 16) || param_size != sizeof(uint8_t)) { @@ -3532,11 +3571,12 @@ int scsipi_ibmtape_get_xattr(void *device, const char *name, char **buf) if (priv->fetch_sec_acq_loss_w == 0 || ((priv->fetch_sec_acq_loss_w + 60 < now.tv_sec) && priv->dirty_acq_loss_w)) { - ret = _cdb_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, logdata, LOGSENSEPAGE); + ret = scsipi_ibmtape_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, logdata, LOGSENSEPAGE); if (ret < 0) { ltfsmsg(LTFS_INFO, 30234I, LOG_PERFORMANCE, ret, "get xattr"); } else { + ret = 0; if (_parse_logPage(logdata, PERF_ACTIVE_CQ_LOSS_W, ¶m_size, logbuf, 16)) { ltfsmsg(LTFS_INFO, 30235I, LOG_PERFORMANCE, "get xattr"); ret = -LTFS_NO_XATTR; @@ -3782,8 +3822,8 @@ int scsipi_ibmtape_get_eod_status(void *device, int part) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_GETEODSTAT)); /* Issue LogPage 0x17 */ - ret = scsipi_ibmtape_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); - if (ret) { + ret = scsipi_ibmtape_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (ret < 0) { ltfsmsg(LTFS_WARN, 30237W, LOG_VOLUMESTATS, ret); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_GETEODSTAT)); return EOD_UNKNOWN; @@ -4465,14 +4505,14 @@ int scsipi_ibmtape_set_profiler(void *device, char *work_dir, bool enable) return rc; } -int scsipi_ibmtape_get_block_in_buffer(void *device, uint32_t *block) +int scsipi_ibmtape_get_next_block_to_xfer(void *device, struct tc_position *pos) { int ret = -EDEV_UNKNOWN; int ret_ep = DEVICE_GOOD; struct scsipi_ibmtape_data *priv = (struct scsipi_ibmtape_data*)device; scsireq_t req; - unsigned char cdb[CDB6_LEN]; + unsigned char cdb[CDB10_LEN]; int timeout; char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "READPOS"; char *msg = NULL; @@ -4480,6 +4520,8 @@ int scsipi_ibmtape_get_block_in_buffer(void *device, uint32_t *block) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_READPOS)); + memset(pos, 0, sizeof(struct tc_position)); + /* Zero out the CDB and the result buffer */ ret = init_scsireq(&req); if (ret < 0) @@ -4490,6 +4532,7 @@ int scsipi_ibmtape_get_block_in_buffer(void *device, uint32_t *block) /* Build CDB */ cdb[0] = READ_POSITION; cdb[1] = 0x08; /* Extended Format */ + ltfs_u16tobe(cdb + 7, sizeof(buf)); /* allocation length */ timeout = ibm_tape_get_timeout(priv->timeouts, cdb[0]); if (timeout < 0) @@ -4505,10 +4548,11 @@ int scsipi_ibmtape_get_block_in_buffer(void *device, uint32_t *block) ret = scsipi_issue_cdb_command(&priv->dev, &req, cmd_desc, &msg); if (ret == DEVICE_GOOD) { - *block = (buf[5] << 16) + (buf[6] << 8) + (int)buf[7]; + pos->partition = (tape_partition_t)buf[1]; + pos->block = ltfs_betou64(buf + 16); - ltfsmsg(LTFS_DEBUG, 30398D, "blocks-in-buffer", - (unsigned long long)*block, (unsigned long long)0, (unsigned long long)0, priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30398D, "next-block-to-xfer", + (unsigned long long)pos->partition, (unsigned long long)pos->block, (unsigned long long)0, priv->drive_serial); } else { ret_ep = _process_errors(device, ret, msg, cmd_desc, true, true); if (ret_ep < 0) @@ -4552,6 +4596,8 @@ struct tape_ops scsipi_ibmtape_handler = { .write_attribute = scsipi_ibmtape_write_attribute, .read_attribute = scsipi_ibmtape_read_attribute, .allow_overwrite = scsipi_ibmtape_allow_overwrite, + .grao = scsipi_ibmtape_grao, + .rrao = scsipi_ibmtape_rrao, // May be command combination .set_compression = scsipi_ibmtape_set_compression, .set_default = scsipi_ibmtape_set_default, @@ -4574,7 +4620,7 @@ struct tape_ops scsipi_ibmtape_handler = { .get_serialnumber = scsipi_ibmtape_get_serialnumber, .get_info = scsipi_ibmtape_get_info, .set_profiler = scsipi_ibmtape_set_profiler, - .get_block_in_buffer = scsipi_ibmtape_get_block_in_buffer, + .get_next_block_to_xfer = scsipi_ibmtape_get_next_block_to_xfer, .is_readonly = scsipi_ibmtape_is_readonly, }; diff --git a/src/tape_drivers/osx/iokit/Makefile.am b/src/tape_drivers/osx/iokit/Makefile.am index 172f00d6..87d80818 100644 --- a/src/tape_drivers/osx/iokit/Makefile.am +++ b/src/tape_drivers/osx/iokit/Makefile.am @@ -41,7 +41,7 @@ AM_LIBTOOLFLAGS = --tag=disable-static libtape_iokit_la_SOURCES = iokit_tape.c iokit_scsi.c iokit_service.c vendor_compat.c ibm_tape.c hp_tape.c quantum_tape.c libtape_iokit_la_DEPENDENCIES = ../../../../messages/libtape_iokit_dat.a ../../../libltfs/libltfs.la ./libtape_iokit_la-reed_solomon_crc.lo ./libtape_iokit_la-crc32c_crc.lo libtape_iokit_la_LIBADD = ../../../libltfs/libltfs.la ./libtape_iokit_la-reed_solomon_crc.lo ./libtape_iokit_la-crc32c_crc.lo -libtape_iokit_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ -L../../../../messages -ltape_iokit_dat +libtape_iokit_la_LDFLAGS = -avoid-version -module @AM_LDFLAGS@ ../../../../messages/libtape_iokit_dat.a libtape_iokit_la_CPPFLAGS = @AM_CPPFLAGS@ @AM_EXTRA_CPPFLAGS@ -I ../../.. -I ../.. vendor_compat.c: diff --git a/src/tape_drivers/osx/iokit/iokit_tape.c b/src/tape_drivers/osx/iokit/iokit_tape.c index fd230675..680615ab 100644 --- a/src/tape_drivers/osx/iokit/iokit_tape.c +++ b/src/tape_drivers/osx/iokit/iokit_tape.c @@ -96,12 +96,16 @@ struct iokit_global_data global_data; #define LOG_PAGE_PARAM_OFFSET (4) #define IOKIT_MAX_BLOCK_SIZE (1 * MB) + +#ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif /* Forward references (For keep function order to struct tape_ops) */ int iokit_readpos(void *device, struct tc_position *pos); int iokit_locate(void *device, struct tc_position dest, struct tc_position *pos); -int iokit_logsense(void *device, const unsigned char page, unsigned char *buf, const size_t size); +int iokit_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size); int iokit_modesense(void *device, const unsigned char page, const TC_MP_PC_TYPE pc, const unsigned char subpage, unsigned char *buf, const size_t size); int iokit_modeselect(void *device, unsigned char *buf, const size_t size); @@ -140,7 +144,6 @@ static inline int _parse_logPage(const unsigned char *logdata, i += param_len + LOG_PAGE_PARAM_OFFSET; } -out: return ret; } @@ -455,7 +458,7 @@ static int _cdb_read_buffer(void *device, int id, unsigned char *buf, size_t off ltfsmsg(LTFS_DEBUG, 30993D, "read buffer", id, priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -507,7 +510,7 @@ static int _cdb_force_dump(struct iokit_data *priv) ltfsmsg(LTFS_DEBUG, 30993D, "force dump", 0, priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); memset(&buf, 0, sizeof(buf)); @@ -731,6 +734,51 @@ static int _register_key(void *device, unsigned char *key) return ret; } +/** SCSI command handling of REPORT SUPPORTED OPERATION CODES + */ +static int _cdb_rsoc(void* device, unsigned char *buf, uint32_t len) +{ + int ret = -EDEV_UNKNOWN; + struct iokit_data *priv = (struct iokit_data*)device; + + struct iokit_scsi_request req; + unsigned char cdb[CDB12_LEN]; + unsigned char sense[MAXSENSE]; + int timeout = 60; + char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "RSOC"; + char *msg = NULL; + + /* Zero out the CDB and the result buffer */ + memset(cdb, 0, sizeof(cdb)); + memset(&req, 0, sizeof(struct iokit_scsi_request)); + memset(buf, 0, len); + memset(sense, 0, sizeof(sense)); + + /* Build CDB */ + cdb[0] = MAINTENANCE_IN; + cdb[1] = 0x0C; /* REPORT SUPPORTED OPERATION CODES */ + cdb[2] = 0x80; /* Fetch all commands with RCTD */ + ltfs_u32tobe(cdb + 6, len); /* allocation length */ + + /* Build request */ + req.dxfer_direction = SCSI_FROM_TARGET_TO_INITIATOR; + req.cmd_len = sizeof(cdb); + req.mx_sb_len = sizeof(SCSI_Sense_Data); + req.dxfer_len = len; + req.dxferp = buf; + req.cmdp = cdb; + memset(&req.sense_buffer, 0, req.mx_sb_len); + req.timeout = IOKitConversion(timeout); + req.desc = cmd_desc; + + ret = iokit_issue_cdb_command(&priv->dev, &req, &msg); + if (ret < 0){ + _process_errors(device, ret, msg, cmd_desc, true); + } + + return ret; +} + /* Global functions */ int iokit_open(const char *devname, void **handle) { @@ -742,6 +790,9 @@ int iokit_open(const char *devname, void **handle) struct iokit_data *priv; scsi_device_identifier id_data; + unsigned char *rsoc_buf = NULL; + uint32_t rsoc_len = RSOC_BUF_SIZE; + ltfsmsg(LTFS_INFO, 30810I, devname); priv = calloc(1, sizeof(struct iokit_data)); @@ -805,14 +856,12 @@ int iokit_open(const char *devname, void **handle) } strncpy(priv->drive_serial, id_data.unit_serial, UNIT_SERIAL_LENGTH - 1); - /* Convert vendor_id in inq buffer to integer based id */ - priv->vendor = get_vendor_id(id_data.vendor_id); - /* Check the drive is supportable */ - struct supported_device **cur = get_supported_devs(priv->vendor); + struct supported_device **cur = get_supported_devs(id_data.vendor_id); while(cur && *cur) { if((! strncmp(id_data.vendor_id, (*cur)->vendor_id, strlen((*cur)->vendor_id)) ) && (! strncmp(id_data.product_id, (*cur)->product_id, strlen((*cur)->product_id)) ) ) { + priv->vendor = (*cur)->vendor_type; drive_type = (*cur)->drive_type; break; } @@ -850,9 +899,44 @@ int iokit_open(const char *devname, void **handle) priv->info.target = 0; priv->info.lun = -1; - /* Setup vendor specific parameters */ + /* Setup error table sense to ltfs error code */ init_error_table(priv->vendor, &standard_table, &vendor_table); - init_timeout(priv->vendor, &priv->timeouts, priv->drive_type); + + /* Setup device specific timeout value */ + rsoc_buf = calloc(1, RSOC_BUF_SIZE); + if (rsoc_buf) { + ret = _cdb_rsoc(&priv->dev, rsoc_buf, RSOC_BUF_SIZE); + rsoc_len = ltfs_betou32(rsoc_buf); + if (!ret && rsoc_len < RSOC_BUF_SIZE) { + ltfsmsg(LTFS_INFO, 30872I, "RSOC"); + ret = init_timeout_rsoc(&priv->timeouts, rsoc_buf, rsoc_len); + } + + if (ret < 0) { + /* + * The drive doesn't support RSOC, buffer overrun or parse error + * try to initialize the timeout table from drive vendor and drive type + */ + ltfsmsg(LTFS_INFO, 30872I, "vendor and device"); + ret = init_timeout(priv->vendor, &priv->timeouts, priv->drive_type); + if (!priv->timeouts) { + ltfsmsg(LTFS_INFO, 30872I, "device"); + ibm_tape_init_timeout(&priv->timeouts, priv->drive_type); + } + } + free(rsoc_buf); + } else { + /* + * Memory allocation failure, try to initialize the timeout table + * from drive vendor and drive type + */ + ltfsmsg(LTFS_INFO, 30872I, "vendor and device"); + init_timeout(priv->vendor, &priv->timeouts, priv->drive_type); + if (!priv->timeouts) { + ltfsmsg(LTFS_INFO, 30872I, "device"); + ibm_tape_init_timeout(&priv->timeouts, priv->drive_type); + } + } /* Register reservation key */ ibm_tape_genkey(priv->key); @@ -880,7 +964,6 @@ int iokit_open(const char *devname, void **handle) int iokit_reopen(const char *devname, void *device) { - char *end; int drive_type = DRIVE_UNSUPPORTED; int ret = -EDEV_UNKNOWN; @@ -931,7 +1014,7 @@ int iokit_reopen(const char *devname, void *device) } else priv->drive_type = drive_type; } else { - ltfsmsg(LTFS_INFO, 30813I, id_data.product_id); + ltfsmsg(LTFS_INFO, 30813I, id_data.vendor_id, id_data.product_id); iokit_release_exclusive_access(&priv->dev); ret = -EDEV_DEVICE_UNSUPPORTABLE; /* Unsupported device */ } @@ -1011,7 +1094,7 @@ int iokit_inquiry_page(void *device, unsigned char page, struct tc_inq_page *inq ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_INQUIRYPAGE)); ltfsmsg(LTFS_DEBUG, 30993D, "inquiry", page, priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1090,7 +1173,7 @@ int iokit_test_unit_ready(void *device) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_TUR)); ltfsmsg(LTFS_DEBUG3, 30992D, "test unit ready", priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1154,7 +1237,7 @@ static int _cdb_read(void *device, char *buf, size_t size, boolean_t sili) char *msg = NULL; size_t length = -EDEV_UNKNOWN; - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1202,10 +1285,10 @@ static int _cdb_read(void *device, char *buf, size_t size, boolean_t sili) * In this case, LTFS will trust SCSI sense. */ if (diff_len < 0) { - ltfsmsg(LTFS_INFO, 30820I, diff_len, size - diff_len); // "Detect overrun condition" + ltfsmsg(LTFS_INFO, 30820I, diff_len, (int)(size - diff_len)); // "Detect overrun condition" ret = -EDEV_OVERRUN; } else { - ltfsmsg(LTFS_DEBUG, 30821D, diff_len, size - diff_len); // "Detect underrun condition" + ltfsmsg(LTFS_DEBUG, 30821D, diff_len, (int)(size - diff_len)); // "Detect underrun condition" length = size - diff_len; ret = DEVICE_GOOD; } @@ -1215,10 +1298,10 @@ static int _cdb_read(void *device, char *buf, size_t size, boolean_t sili) #endif } else { if (diff_len < 0) { - ltfsmsg(LTFS_INFO, 30820I, diff_len, size - diff_len); // "Detect overrun condition" + ltfsmsg(LTFS_INFO, 30820I, diff_len, (int)(size - diff_len)); // "Detect overrun condition" ret = -EDEV_OVERRUN; } else { - ltfsmsg(LTFS_DEBUG, 30821D, diff_len, size - diff_len); // "Detect underrun condition" + ltfsmsg(LTFS_DEBUG, 30821D, diff_len, (int)(size - diff_len)); // "Detect underrun condition" length = size - diff_len; ret = DEVICE_GOOD; } @@ -1301,7 +1384,8 @@ int iokit_read(void *device, char *buf, size_t size, return ret; } goto read_retry; - } else if ( !(pos->block) && unusual_size && (ret == size || ret == -EDEV_FILEMARK_DETECTED) ) { + } else if ( !(pos->block) && unusual_size && + (ret == (int32_t)size || ret == -EDEV_FILEMARK_DETECTED) ) { /* * Try to read again without sili bit, because some I/F doesn't support SILION read correctly * like @@ -1354,7 +1438,7 @@ static int _cdb_write(void *device, uint8_t *buf, size_t size, bool *ew, bool *p char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "WRITE"; char *msg = NULL; - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1474,7 +1558,7 @@ int iokit_writefm(void *device, size_t count, struct tc_position *pos, bool imme ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_WRITEFM)); ltfsmsg(LTFS_DEBUG, 30994D, "write file marks", count, priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1552,9 +1636,9 @@ int iokit_rewind(void *device, struct tc_position *pos) char *msg = NULL; ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_REWIND)); - ltfsmsg(LTFS_DEBUG, 30997D, "rewind", 0, 0, priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30997D, "rewind", (unsigned long long)0, (unsigned long long)0, priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1615,7 +1699,7 @@ int iokit_locate(void *device, struct tc_position dest, struct tc_position *pos) bool pc = false; ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOCATE)); - ltfsmsg(LTFS_DEBUG, 30997D, "locate", dest.partition, dest.block, priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30997D, "locate", (unsigned long long)dest.partition, dest.block, priv->drive_serial); if (pos->partition != dest.partition) { if (priv->clear_by_pc) { @@ -1629,7 +1713,7 @@ int iokit_locate(void *device, struct tc_position dest, struct tc_position *pos) pc = true; } - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1692,7 +1776,7 @@ int iokit_space(void *device, size_t count, TC_SPACE_TYPE type, struct tc_positi ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_SPACE)); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1775,7 +1859,7 @@ static int _cdb_request_sense(void *device, unsigned char *buf, unsigned char si char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "REQUEST_SENSE"; char *msg = NULL; - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1825,7 +1909,7 @@ int iokit_erase(void *device, struct tc_position *pos, bool long_erase) get_current_timespec(&ts_start); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1868,7 +1952,7 @@ int iokit_erase(void *device, struct tc_position *pos, bool long_erase) if (IS_ENTERPRISE(priv->drive_type)) { get_current_timespec(&ts_now); - ltfsmsg(LTFS_INFO, 30829I, (ts_now.tv_sec - ts_start.tv_sec)/60); + ltfsmsg(LTFS_INFO, 30829I, (int)((ts_now.tv_sec - ts_start.tv_sec)/60)); } else { progress = ((uint32_t) sense_buf[16] & 0xFF) << 8; progress += ((uint32_t) sense_buf[17] & 0xFF); @@ -1899,7 +1983,7 @@ static int _cdb_load_unload(void *device, bool load) char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "LOAD_UNLOAD"; char *msg = NULL; - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -1980,14 +2064,26 @@ int iokit_load(void *device, struct tc_position *pos) priv->density_code = buf[8]; - if (priv->vendor == VENDOR_HP) { + if (buf[2] == 0x00 || buf[2] == 0x01) { + /* + * Non-IBM drive doesn't have cartridge type so need to assume from density code. + */ priv->cart_type = assume_cart_type(priv->density_code); if (buf[2] == 0x01) priv->is_worm = true; } else { + /* + * IBM drive haves cartridge type in buf[2] like TC_MP_LTO5D_CART. + */ priv->cart_type = buf[2]; } + if (priv->cart_type == 0x00) { + ltfsmsg(LTFS_WARN, 30871W); + ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOAD)); + return 0; + } + ret = is_supported_tape(priv->cart_type, priv->density_code, &(priv->is_worm)); if(ret == -LTFS_UNSUPPORTED_MEDIUM) ltfsmsg(LTFS_INFO, 30831I, priv->cart_type, priv->density_code); @@ -2030,7 +2126,7 @@ int iokit_readpos(void *device, struct tc_position *pos) struct iokit_data *priv = (struct iokit_data*)device; struct iokit_scsi_request req; - unsigned char cdb[CDB6_LEN]; + unsigned char cdb[CDB10_LEN]; int timeout; char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "READPOS"; char *msg = NULL; @@ -2038,7 +2134,7 @@ int iokit_readpos(void *device, struct tc_position *pos) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_READPOS)); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2117,7 +2213,7 @@ int iokit_setcap(void *device, uint16_t proportion) ret = iokit_modeselect(device, buf, sizeof(buf)); } else { - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2163,7 +2259,7 @@ int iokit_format(void *device, TC_FORMAT_TYPE format, const char *vol_name, cons ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_FORMAT)); ltfsmsg(LTFS_DEBUG, 30992D, "format", priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2216,9 +2312,12 @@ int iokit_remaining_capacity(void *device, struct tc_remaining_cap *cap) memset(&buffer, 0, LOGSENSEPAGE); - if (IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x05)) { + if ((IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x05)) || + (priv->vendor == VENDOR_HP && IS_LTO(priv->drive_type) && (DRIVE_GEN(priv->drive_type) == 0x06)) || + (priv->vendor == VENDOR_QUANTUM_B && IS_LTO(priv->drive_type))) { + /* Use LogPage 0x31 */ - ret = iokit_logsense(device, (uint8_t)LOG_TAPECAPACITY, (void *)buffer, LOGSENSEPAGE); + ret = iokit_logsense(device, (uint8_t)LOG_TAPECAPACITY, (uint8_t)0, (void *)buffer, LOGSENSEPAGE); if(ret < 0) { ltfsmsg(LTFS_INFO, 30832I, LOG_VOLUMESTATS, ret); @@ -2261,9 +2360,8 @@ int iokit_remaining_capacity(void *device, struct tc_remaining_cap *cap) ret = DEVICE_GOOD; } else { /* Use LogPage 0x17 */ - ret = iokit_logsense(device, LOG_VOLUMESTATS, (void *)buffer, LOGSENSEPAGE); - if(ret < 0) - { + ret = iokit_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, (void *)buffer, LOGSENSEPAGE); + if(ret < 0) { ltfsmsg(LTFS_INFO, 30832I, LOG_VOLUMESTATS, ret); goto out; } @@ -2281,7 +2379,7 @@ int iokit_remaining_capacity(void *device, struct tc_remaining_cap *cap) offset = (int)buf[0] + 1; length = (int)buf[offset] + 1; - if (offset + length <= param_size) { + if ((uint32_t)(offset + length) <= param_size) { cap->max_p1 = ltfs_betou32(&buf[offset + PARTITIOIN_REC_HEADER_LEN]); } @@ -2296,7 +2394,7 @@ int iokit_remaining_capacity(void *device, struct tc_remaining_cap *cap) offset = (int)buf[0] + 1; length = (int)buf[offset] + 1; - if (offset + length <= param_size) { + if ((uint32_t)(offset + length) <= param_size) { cap->remaining_p1 = ltfs_betou32(&buf[offset + PARTITIOIN_REC_HEADER_LEN]); } @@ -2319,8 +2417,8 @@ int iokit_remaining_capacity(void *device, struct tc_remaining_cap *cap) return ret; } -static int _cdb_logsense(void *device, const unsigned char page, const unsigned char subpage, - unsigned char *buf, const size_t size) +int iokit_logsense(void *device, const uint8_t page, const uint8_t subpage, + unsigned char *buf, const size_t size) { int ret = -EDEV_UNKNOWN; struct iokit_data *priv = (struct iokit_data*)device; @@ -2331,9 +2429,18 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "LOGSENSE"; char *msg = NULL; + unsigned int len = 0; + unsigned char *inner_buf = NULL; + ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_LOGSENSE)); + ltfsmsg(LTFS_DEBUG3, 30997D, "logsense", + (unsigned long long)page, (unsigned long long)subpage, priv->drive_serial); - // Zero out the CDB and the result buffer + inner_buf = calloc(1, MAXLP_SIZE); /* Assume max length of LP is 0xFFFF */ + if (!inner_buf) + return -LTFS_NO_MEMORY; + + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2341,18 +2448,20 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned cdb[0] = LOG_SENSE; cdb[2] = 0x40 | (page & 0x3F); /* Current value */ cdb[3] = subpage; - ltfs_u16tobe(cdb + 7, size); + ltfs_u16tobe(cdb + 7, MAXLP_SIZE); timeout = get_timeout(priv->timeouts, cdb[0]); - if (timeout < 0) + if (timeout < 0) { + free(inner_buf); return -EDEV_UNSUPPORETD_COMMAND; + } /* Build request */ req.dxfer_direction = SCSI_FROM_TARGET_TO_INITIATOR; req.cmd_len = sizeof(cdb); req.mx_sb_len = sizeof(SCSI_Sense_Data); - req.dxfer_len = size; - req.dxferp = buf; + req.dxfer_len = MAXLP_SIZE; + req.dxferp = inner_buf; req.cmdp = cdb; memset(&req.sense_buffer, 0, req.mx_sb_len); req.timeout = IOKitConversion(timeout); @@ -2361,19 +2470,19 @@ static int _cdb_logsense(void *device, const unsigned char page, const unsigned ret = iokit_issue_cdb_command(&priv->dev, &req, &msg); if (ret < 0){ _process_errors(device, ret, msg, cmd_desc, true); - } - - ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); + } else { + len = ((int)inner_buf[2] << 8) + (int)inner_buf[3] + 4; - return ret; -} + if (size > len) + memcpy(buf, inner_buf, len); + else + memcpy(buf, inner_buf, size); -int iokit_logsense(void *device, const unsigned char page, unsigned char *buf, const size_t size) -{ - int ret = -EDEV_UNKNOWN; + ret = len; + } - ltfsmsg(LTFS_DEBUG3, 30993D, "logsense", page, ""); - ret = _cdb_logsense(device, page, 0x00, buf, size); + free (inner_buf); + ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_LOGSENSE)); return ret; } @@ -2393,7 +2502,7 @@ int iokit_modesense(void *device, const unsigned char page, const TC_MP_PC_TYPE ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_MODESENSE)); ltfsmsg(LTFS_DEBUG3, 30993D, "modesense", page, priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2444,7 +2553,7 @@ int iokit_modeselect(void *device, unsigned char *buf, const size_t size) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_MODESELECT)); ltfsmsg(LTFS_DEBUG3, 30992D, "modeselect", priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2493,7 +2602,7 @@ int iokit_reserve(void *device) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_RESERVEUNIT)); ltfsmsg(LTFS_DEBUG, 30992D, "reserve unit (6)", priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2525,7 +2634,7 @@ int iokit_reserve(void *device) int count = 0; ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_RESERVEUNIT)); - ltfsmsg(LTFS_DEBUG, 30392D, "reserve (PRO)", priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30992D, "reserve (PRO)", priv->drive_serial); start: ret = _cdb_pro(device, PRO_ACT_RESERVE, PRO_TYPE_EXCLUSIVE, @@ -2565,7 +2674,7 @@ int iokit_release(void *device) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_RELEASEUNIT)); ltfsmsg(LTFS_DEBUG, 30992D, "release unit (6)", priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2595,7 +2704,7 @@ int iokit_release(void *device) #else /* Use persistent reserve */ ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_RELEASEUNIT)); - ltfsmsg(LTFS_DEBUG, 30392D, "release (PRO)", priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30992D, "release (PRO)", priv->drive_serial); ret = _cdb_pro(device, PRO_ACT_RELEASE, PRO_TYPE_EXCLUSIVE, priv->key, NULL); @@ -2618,7 +2727,7 @@ static int _cdb_prevent_allow_medium_removal(void *device, bool prevent) char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "PREVENT/ALLOW_MEDIUM_REMOVAL"; char *msg = NULL; - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2700,7 +2809,7 @@ int iokit_write_attribute(void *device, const tape_partition_t part, ltfs_u32tobe(buffer, len); memcpy(buffer + 4, buf, size); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2752,17 +2861,22 @@ int iokit_read_attribute(void *device, const tape_partition_t part, char *msg = NULL; ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_READATTR)); - ltfsmsg(LTFS_DEBUG3, 30997D, "readattr", (unsigned long)part, id, priv->drive_serial); + ltfsmsg(LTFS_DEBUG3, 30997D, "readattr", (unsigned long long)part, (unsigned long long)id, priv->drive_serial); /* Prepare the buffer to transfer */ - uint32_t len = size + 4; + uint32_t len = 0; + if (size == MAXMAM_SIZE) + len = MAXMAM_SIZE; + else + len = size + 4; + unsigned char *buffer = calloc(1, len); if (!buffer) { ltfsmsg(LTFS_ERR, 10001E, __FUNCTION__); return -EDEV_NO_MEMORY; } - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2807,7 +2921,12 @@ int iokit_read_attribute(void *device, const tape_partition_t part, id != TC_MAM_APP_FORMAT_VERSION) ltfsmsg(LTFS_INFO, 30836I, ret); } else { - memcpy(buf, buffer + 4, size); + if (size == MAXMAM_SIZE) { + /* Include header if request size is MAXMAM_SIZE */ + memcpy(buf, buffer, size); + } else { + memcpy(buf, buffer + 4, size); + } } free(buffer); @@ -2828,9 +2947,9 @@ int iokit_allow_overwrite(void *device, const struct tc_position pos) char *msg = NULL; ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_ALLOWOVERW)); - ltfsmsg(LTFS_DEBUG, 30997D, "allow overwrite", pos.partition, pos.block, priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30997D, "allow overwrite", (unsigned long long)pos.partition, pos.block, priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -2867,6 +2986,24 @@ int iokit_allow_overwrite(void *device, const struct tc_position pos) return ret; } +/** + * GRAO command is currently unsupported on this device + */ +int iokit_grao(void *device, unsigned char *buf, const uint32_t len) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + +/** + * RRAO command is currently unsupported on this device + */ +int iokit_rrao(void *device, unsigned char *buf, const uint32_t len, size_t *out_size) +{ + int ret = -EDEV_UNSUPPORETD_COMMAND; + return ret; +} + int iokit_set_compression(void *device, const bool enable_compression, struct tc_position *pos) { int ret = -EDEV_UNKNOWN; @@ -2995,8 +3132,8 @@ int iokit_get_cartridge_health(void *device, struct tc_cartridge_health *cart_he /* Issue LogPage 0x37 */ cart_health->tape_efficiency = UNSUPPORTED_CARTRIDGE_HEALTH; - ret = iokit_logsense(device, LOG_PERFORMANCE, logdata, LOGSENSEPAGE); - if (ret) + ret = iokit_logsense(device, LOG_PERFORMANCE, (uint8_t)0, logdata, LOGSENSEPAGE); + if (ret < 0) ltfsmsg(LTFS_INFO, 30837I, LOG_PERFORMANCE, ret, "get cart health"); else { for(i = 0; i < (int)((sizeof(perfstats)/sizeof(perfstats[0]))); i++) { @@ -3050,7 +3187,7 @@ int iokit_get_cartridge_health(void *device, struct tc_cartridge_health *cart_he cart_health->read_mbytes = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_begin = UNSUPPORTED_CARTRIDGE_HEALTH; cart_health->passes_middle = UNSUPPORTED_CARTRIDGE_HEALTH; - ret = iokit_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); + ret = iokit_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); if (ret < 0) ltfsmsg(LTFS_INFO, 30837I, LOG_VOLUMESTATS, ret, "get cart health"); else { @@ -3146,10 +3283,11 @@ int iokit_get_tape_alert(void *device, uint64_t *tape_alert) /* Issue LogPage 0x2E */ ta = 0; - ret = iokit_logsense(device, LOG_TAPE_ALERT, logdata, LOGSENSEPAGE); + ret = iokit_logsense(device, LOG_TAPE_ALERT, (uint8_t)0, logdata, LOGSENSEPAGE); if (ret < 0) ltfsmsg(LTFS_INFO, 30837I, LOG_TAPE_ALERT, ret, "get tape alert"); else { + ret = 0; for(i = 1; i <= 64; i++) { if (_parse_logPage(logdata, (uint16_t) i, ¶m_size, buf, 16) || param_size != sizeof(uint8_t)) { @@ -3202,11 +3340,11 @@ int iokit_get_xattr(void *device, const char *name, char **buf) if (priv->fetch_sec_acq_loss_w == 0 || ((priv->fetch_sec_acq_loss_w + 60 < now.tv_sec) && priv->dirty_acq_loss_w)) { - ret = _cdb_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, logdata, LOGSENSEPAGE); - + ret = iokit_logsense(device, LOG_PERFORMANCE, LOG_PERFORMANCE_CAPACITY_SUB, logdata, LOGSENSEPAGE); if (ret < 0) { ltfsmsg(LTFS_INFO, 30837I, LOG_PERFORMANCE, ret, "get xattr"); } else { + ret = 0; if (_parse_logPage(logdata, PERF_ACTIVE_CQ_LOSS_W, ¶m_size, logbuf, 16)) { ltfsmsg(LTFS_INFO, 30838I, LOG_PERFORMANCE, "get xattr"); ret = -LTFS_NO_XATTR; @@ -3306,7 +3444,7 @@ static int _cdb_read_block_limits(void *device) { ltfsmsg(LTFS_DEBUG, 30992D, "read block limits", priv->drive_serial); - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -3434,8 +3572,8 @@ int iokit_get_eod_status(void *device, int part) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_GETEODSTAT)); /* Issue LogPage 0x17 */ - ret = iokit_logsense(device, LOG_VOLUMESTATS, logdata, LOGSENSEPAGE); - if (ret) { + ret = iokit_logsense(device, LOG_VOLUMESTATS, (uint8_t)0, logdata, LOGSENSEPAGE); + if (ret < 0) { ltfsmsg(LTFS_WARN, 30840W, LOG_VOLUMESTATS, ret); ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_GETEODSTAT)); return EOD_UNKNOWN; @@ -3507,7 +3645,7 @@ int iokit_get_device_list(struct tc_drive_info *buf, int count) int i, ret; int found = 0; int32_t devs = iokit_get_ssc_device_count(); - uint32_t drive_type; + int drive_type; scsi_device_identifier identifier; struct iokit_device *iokit_device; // Pointer to device status structure @@ -3526,7 +3664,7 @@ int iokit_get_device_list(struct tc_drive_info *buf, int count) continue; } drive_type = iokit_get_drive_identifier(iokit_device, &identifier); - if (drive_type != -1) { + if (!drive_type) { if (found < count && buf) { snprintf(buf[i].name, TAPE_DEVNAME_LEN_MAX, "%d", i); snprintf(buf[i].vendor, TAPE_VENDOR_NAME_LEN_MAX, "%s", identifier.vendor_id); @@ -3599,7 +3737,7 @@ static int _cdb_spin(void *device, const uint16_t sps, unsigned char **buffer, s char *msg = NULL; size_t len = *size + 4; - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -3652,7 +3790,7 @@ int _cdb_spout(void *device, const uint16_t sps, char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "SPOUT"; char *msg = NULL; - // Zero out the CDB and the result buffer + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); memset(&req, 0, sizeof(struct iokit_scsi_request)); @@ -3956,7 +4094,6 @@ int iokit_get_keyalias(void *device, unsigned char **keyalias) free: free(buffer); -out: ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_EXIT(REQ_TC_GETKEYALIAS)); return ret; } @@ -4101,13 +4238,13 @@ int iokit_set_profiler(void *device, char *work_dir, bool enable) return rc; } -int iokit_get_block_in_buffer(void *device, uint32_t *block) +int iokit_get_next_block_to_xfer(void *device, struct tc_position *pos) { int ret = -EDEV_UNKNOWN; struct iokit_data *priv = (struct iokit_data*)device; struct iokit_scsi_request req; - unsigned char cdb[CDB6_LEN]; + unsigned char cdb[CDB10_LEN]; int timeout; char cmd_desc[COMMAND_DESCRIPTION_LENGTH] = "READPOS"; char *msg = NULL; @@ -4115,12 +4252,16 @@ int iokit_get_block_in_buffer(void *device, uint32_t *block) ltfs_profiler_add_entry(priv->profiler, NULL, TAPEBEND_REQ_ENTER(REQ_TC_READPOS)); + memset(pos, 0, sizeof(struct tc_position)); + /* Zero out the CDB and the result buffer */ memset(cdb, 0, sizeof(cdb)); + memset(&req, 0, sizeof(struct iokit_scsi_request)); /* Build CDB */ cdb[0] = READ_POSITION; cdb[1] = 0x08; /* Extended Format */ + ltfs_u16tobe(cdb + 7, sizeof(buf)); /* allocation length */ timeout = get_timeout(priv->timeouts, cdb[0]); if (timeout < 0) @@ -4139,10 +4280,11 @@ int iokit_get_block_in_buffer(void *device, uint32_t *block) ret = iokit_issue_cdb_command(&priv->dev, &req, &msg); if (ret == DEVICE_GOOD) { - *block = (buf[5] << 16) + (buf[6] << 8) + (int)buf[7]; + pos->partition = (tape_partition_t)buf[1]; + pos->block = ltfs_betou64(buf + 16); - ltfsmsg(LTFS_DEBUG, 30998D, "blocks-in-buffer", - (unsigned long long) *block, 0, 0, priv->drive_serial); + ltfsmsg(LTFS_DEBUG, 30998D, "next-block-to-xfer", + (unsigned long long)pos->partition, (unsigned long long)pos->block, (unsigned long long)0, priv->drive_serial); } else { _process_errors(device, ret, msg, cmd_desc, true); } @@ -4184,6 +4326,8 @@ struct tape_ops iokit_handler = { .write_attribute = iokit_write_attribute, .read_attribute = iokit_read_attribute, .allow_overwrite = iokit_allow_overwrite, + .grao = iokit_grao, + .rrao = iokit_rrao, // May be command combination .set_compression = iokit_set_compression, .set_default = iokit_set_default, @@ -4206,7 +4350,7 @@ struct tape_ops iokit_handler = { .get_serialnumber = iokit_get_serialnumber, .get_info = iokit_get_info, .set_profiler = iokit_set_profiler, - .get_block_in_buffer = iokit_get_block_in_buffer, + .get_next_block_to_xfer = iokit_get_next_block_to_xfer, .is_readonly = iokit_is_readonly, }; diff --git a/src/tape_drivers/quantum_tape.c b/src/tape_drivers/quantum_tape.c index 9f583ac3..8ca47914 100644 --- a/src/tape_drivers/quantum_tape.c +++ b/src/tape_drivers/quantum_tape.c @@ -63,10 +63,12 @@ #include "libltfs/ltfs_endian.h" struct supported_device *quantum_supported_drives[] = { - TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH5", DRIVE_LTO5_HH, "[ULTRIUM-HH5]" ), /* QUANTUM Ultrium Gen 5 Half-High */ - TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH6", DRIVE_LTO6_HH, "[ULTRIUM-HH6]" ), /* QUANTUM Ultrium Gen 6 Half-High */ - TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH7", DRIVE_LTO7_HH, "[ULTRIUM-HH7]" ), /* QUANTUM Ultrium Gen 7 Half-High */ - TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH8", DRIVE_LTO8_HH, "[ULTRIUM-HH8]" ), /* QUANTUM Ultrium Gen 8 Half-High */ + TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH5", VENDOR_QUANTUM, DRIVE_LTO5_HH, "[ULTRIUM-HH5]" ), /* QUANTUM Ultrium Gen 5 Half-High */ + TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH6", VENDOR_QUANTUM, DRIVE_LTO6_HH, "[ULTRIUM-HH6]" ), /* QUANTUM Ultrium Gen 6 Half-High */ + TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH7", VENDOR_QUANTUM, DRIVE_LTO7_HH, "[ULTRIUM-HH7]" ), /* QUANTUM Ultrium Gen 7 Half-High */ + TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM-HH8", VENDOR_QUANTUM, DRIVE_LTO8_HH, "[ULTRIUM-HH8]" ), /* QUANTUM Ultrium Gen 8 Half-High */ + TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM 5", VENDOR_QUANTUM_B, DRIVE_LTO5_HH, "[ULTRIUM-5]" ), /* Another QUANTUM Ultrium Gen 5 Half-High */ + TAPEDRIVE( QUANTUM_VENDOR_ID, "ULTRIUM 6", VENDOR_QUANTUM_B, DRIVE_LTO6_HH, "[ULTRIUM-6]" ), /* Another QUANTUM Ultrium Gen 6 Half-High */ /* End of supported_devices */ NULL }; diff --git a/src/tape_drivers/tape_drivers.h b/src/tape_drivers/tape_drivers.h index 1d087710..8a43a1e0 100644 --- a/src/tape_drivers/tape_drivers.h +++ b/src/tape_drivers/tape_drivers.h @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2023 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -69,10 +69,18 @@ #define REDPOS_LONG_LEN (32) #define REDPOS_EXT_LEN (32) +#define RSOC_BUF_SIZE (4 * KB) +#define RSOC_ENT_SIZE (20) +#define RSOC_HEADER_SIZE (4) +#define RSOC_RECOM_TO_OFFSET (16) + #ifndef MAXSENSE #define MAXSENSE (255) #endif +#define MAXLP_SIZE (0xFFFF) +#define MAXMAM_SIZE (0xFFFF) + #define MASK_WITH_SENSE_KEY (0xFFFFFF) #define MASK_WITHOUT_SENSE_KEY (0x00FFFF) @@ -81,7 +89,8 @@ typedef int (*crc_check)(void *buf, size_t n); typedef void* (*memcpy_crc_enc)(void *dest, const void *src, size_t n); typedef int (*memcpy_crc_check)(void *dest, const void *src, size_t n); -#define THRESHOLD_FORCE_WRITE_NO_WRITE (5) + +#define THRESHOLD_FORCE_WRITE_NO_WRITE (20) #define DEFAULT_WRITEPERM (0) #define DEFAULT_READPERM (0) #define DEFAULT_ERRORTYPE (0) @@ -111,9 +120,9 @@ static inline int _sense2errorcode(uint32_t sense, struct error_table *table, ch if ( (sense & 0xFFFF00) == 0x044000 ) sense = 0x044000; - else if ( (sense & 0xFFF000) == 0x048000 ) /* 04/8xxx in TS3100/TS3200 */ + else if ( (sense & 0xFFFF00) == 0x048000 ) /* 04/8xxx in TS3100/TS3200 */ sense = 0x048000; - else if ( (sense & 0xFFF000) == 0x0B4100 ) /* 0B/41xx in TS2900 */ + else if ( (sense & 0xFFFF00) == 0x0B4100 ) /* 0B/41xx in TS2900 */ sense = 0x0B4100; if ( (sense & 0x00FF00) >= 0x008000 || (sense & 0x0000FF) >= 0x000080) @@ -141,11 +150,12 @@ static inline int _sense2errorcode(uint32_t sense, struct error_table *table, ch struct supported_device { char vendor_id[VENDOR_ID_LENGTH + 1]; char product_id[PRODUCT_ID_LENGTH + 1]; + int vendor_type; int drive_type; char product_name[PRODUCT_NAME_LENGTH + 1]; }; -#define TAPEDRIVE(v, p, t, n) &(struct supported_device){ v, p, t, n } +#define TAPEDRIVE(v, p, vt, dt, n) &(struct supported_device){ v, p, vt, dt, n } #define TAPE_FAMILY_MASK (0xF000) #define TAPE_FAMILY_ENTERPRISE (0x1000) @@ -169,7 +179,8 @@ enum { VENDOR_UNKNOWN = 0, VENDOR_IBM, VENDOR_HP, - VENDOR_QUANTUM, + VENDOR_QUANTUM, /* Quantum model C */ + VENDOR_QUANTUM_B, /* Quantum model B */ }; enum { @@ -188,6 +199,7 @@ enum { DRIVE_TS1150 = 0x1105, /* TS1150 */ DRIVE_TS1155 = 0x5105, /* TS1155 */ DRIVE_TS1160 = 0x1106, /* TS1160 */ + DRIVE_TS1170 = 0x1107, /* TS1170 */ }; enum { @@ -201,6 +213,7 @@ enum { DRIVE_GEN_JAG5 = 0x1005, DRIVE_GEN_JAG5A = 0x5005, DRIVE_GEN_JAG6 = 0x1006, + DRIVE_GEN_JAG7 = 0x1007, }; /* LTO cartridge type in mode page header */ @@ -245,6 +258,8 @@ enum { TC_MP_JE = 0x95, /* IBM TS11x0 JE cartridge */ TC_MP_JV = 0xA5, /* IBM TS11x0 JV cartridge */ TC_MP_JM = 0xB4, /* IBM TS11x0 JM cartridge */ + /* 6th gen */ + TC_MP_JF = 0x96, /* IBM TS11x0 JF cartridge */ }; #define IS_REFORMATTABLE_TAPE(t) \ @@ -259,6 +274,7 @@ enum { t == TC_MP_JE || \ t == TC_MP_JV || \ t == TC_MP_JM || \ + t == TC_MP_JF || \ t == TC_MP_LTO7D_CART ) #endif // __tape_drivers_h diff --git a/src/tape_drivers/vendor_compat.c b/src/tape_drivers/vendor_compat.c index 482479db..7096415b 100644 --- a/src/tape_drivers/vendor_compat.c +++ b/src/tape_drivers/vendor_compat.c @@ -281,35 +281,20 @@ struct error_table standard_tape_errors[] = { {0xFFFFFF, -EDEV_UNKNOWN, "Unknown Error code"}, }; -int get_vendor_id(char* vendor) -{ - if (!strncmp(vendor, IBM_VENDOR_ID, strlen(IBM_VENDOR_ID))) - return VENDOR_IBM; - else if (!strncmp(vendor, HP_VENDOR_ID, strlen(HP_VENDOR_ID))) - return VENDOR_HP; - else if (!strncmp(vendor, HPE_VENDOR_ID, strlen(HPE_VENDOR_ID))) - return VENDOR_HP; - else if (!strncmp(vendor, QUANTUM_VENDOR_ID, strlen(QUANTUM_VENDOR_ID))) - return VENDOR_QUANTUM; - else - return VENDOR_UNKNOWN; -} - -struct supported_device **get_supported_devs(int vendor) +struct supported_device **get_supported_devs(const char *vendor_str) { struct supported_device **cur = NULL; - switch (vendor) { - case VENDOR_IBM: - cur = ibm_supported_drives; - break; - case VENDOR_HP: - cur = hp_supported_drives; - break; - case VENDOR_QUANTUM: - cur = quantum_supported_drives; - break; - } + if (! strncmp(vendor_str, IBM_VENDOR_ID, strlen(IBM_VENDOR_ID))) + cur = ibm_supported_drives; + else if (! strncmp(vendor_str, HP_VENDOR_ID, strlen(HP_VENDOR_ID))) + cur = hp_supported_drives; + else if (! strncmp(vendor_str, HPE_VENDOR_ID, strlen(HPE_VENDOR_ID))) + cur = hp_supported_drives; + else if (! strncmp(vendor_str, TANDBERG_VENDOR_ID, strlen(TANDBERG_VENDOR_ID))) + cur = hp_supported_drives; + else if (! strncmp(vendor_str, QUANTUM_VENDOR_ID, strlen(QUANTUM_VENDOR_ID))) + cur = quantum_supported_drives; return cur; } @@ -428,6 +413,39 @@ int init_timeout(int vendor, struct timeout_tape **table, int type) return ret; } +int init_timeout_rsoc(struct timeout_tape **table, unsigned char *buf, uint32_t len) +{ + int entries = len / RSOC_ENT_SIZE; + int modulo = len % RSOC_ENT_SIZE; + int i = 0, offset = 0; + int op_code, timeout; + struct timeout_tape *entry, *out = NULL; + + if (modulo) { + /* Descriptor length is invalid */ + return -EDEV_INVALID_ARG; + } + + HASH_CLEAR(hh, *table); + + for (i = 0; i < entries; i++) { + offset = RSOC_HEADER_SIZE + (RSOC_ENT_SIZE * i); + op_code = (int)(*((unsigned char *)(buf + offset))); + timeout = (int)(ltfs_betou32(buf + offset + RSOC_RECOM_TO_OFFSET)); + + out = NULL; + HASH_FIND_INT(*table, &op_code, out); + if (!out) { + entry = malloc(sizeof(struct timeout_tape)); + entry->op_code = op_code; + entry->timeout = timeout; + HASH_ADD_INT(*table, op_code, entry); + } + } + + return 0; +} + void destroy_timeout(struct timeout_tape** table) { struct timeout_tape *entry, *tmp; diff --git a/src/tape_drivers/vendor_compat.h b/src/tape_drivers/vendor_compat.h index 08c1e245..19ae377f 100644 --- a/src/tape_drivers/vendor_compat.h +++ b/src/tape_drivers/vendor_compat.h @@ -65,8 +65,7 @@ extern "C" { extern struct error_table standard_tape_errors[]; -int get_vendor_id(char* vendor); -struct supported_device **get_supported_devs(int vendor); +struct supported_device **get_supported_devs(const char *vendor_str); bool drive_has_supported_fw(int vendor, int drive_type, const unsigned char * const revision); unsigned char assume_cart_type(const unsigned char dc); int is_supported_tape(unsigned char type, unsigned char density, bool *is_worm); @@ -76,6 +75,7 @@ void init_error_table(int vendor, struct error_table **vendor_table); int init_timeout(int vendor, struct timeout_tape **table, int type); +int init_timeout_rsoc(struct timeout_tape **table, unsigned char *buf, uint32_t len); void destroy_timeout(struct timeout_tape **table); int get_timeout(struct timeout_tape *table, int op_code); diff --git a/src/utils/.gitignore b/src/utils/.gitignore index 37cf58f7..69d22ee6 100644 --- a/src/utils/.gitignore +++ b/src/utils/.gitignore @@ -1,2 +1,3 @@ ltfsck mkltfs +ltfsindextool diff --git a/src/utils/Makefile.am b/src/utils/Makefile.am index 786b8670..918d8365 100644 --- a/src/utils/Makefile.am +++ b/src/utils/Makefile.am @@ -33,7 +33,7 @@ # OO_Copyright_END # -bin_PROGRAMS = mkltfs ltfsck +bin_PROGRAMS = mkltfs ltfsck ltfsindextool nobase_bin_SCRIPTS = ltfs_ordered_copy @@ -42,11 +42,17 @@ noinst_HEADERS = mkltfs_SOURCES = mkltfs.c mkltfs_DEPENDENCIES = ../libltfs/libltfs.la ../../messages/libbin_mkltfs_dat.a mkltfs_LDADD = ../libltfs/libltfs.la -mkltfs_LDFLAGS = @AM_LDFLAGS@ -L../../messages -lbin_mkltfs_dat +mkltfs_LDFLAGS = @AM_LDFLAGS@ ../../messages/libbin_mkltfs_dat.a mkltfs_CPPFLAGS = @AM_CPPFLAGS@ -I .. -fPIC ltfsck_SOURCES = ltfsck.c ltfsck_DEPENDENCIES = ../libltfs/libltfs.la ../../messages/libbin_ltfsck_dat.a ltfsck_LDADD = ../libltfs/libltfs.la -ltfsck_LDFLAGS = @AM_LDFLAGS@ -L../../messages -lbin_ltfsck_dat +ltfsck_LDFLAGS = @AM_LDFLAGS@ ../../messages/libbin_ltfsck_dat.a ltfsck_CPPFLAGS = @AM_CPPFLAGS@ -I .. -fPIC + +ltfsindextool_SOURCES = ltfsindextool.c +ltfsindextool_DEPENDENCIES = ../libltfs/libltfs.la ../../messages/libbin_ltfsindextool_dat.a ../../messages/libbin_mkltfs_dat.a +ltfsindextool_LDADD = ../libltfs/libltfs.la +ltfsindextool_LDFLAGS = @AM_LDFLAGS@ -L../../messages -lbin_ltfsindextool_dat +ltfsindextool_CPPFLAGS = @AM_CPPFLAGS@ -I .. -fPIC diff --git a/src/utils/ltfs_ordered_copy b/src/utils/ltfs_ordered_copy index 19b5356a..e777ff62 100755 --- a/src/utils/ltfs_ordered_copy +++ b/src/utils/ltfs_ordered_copy @@ -4,7 +4,7 @@ # OO_Copyright_BEGIN # # -# Copyright 2010, 2020 IBM Corp. All rights reserved. +# Copyright 2010, 2021 IBM Corp. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,6 +34,7 @@ # OO_Copyright_END import sys +import errno import platform import os.path import argparse @@ -44,6 +45,11 @@ import threading from logging import getLogger, basicConfig, NOTSET, CRITICAL, ERROR, WARNING, INFO, DEBUG from collections import deque +def is_errno(num, names): + if not num in errno.errorcode: + return False + return errno.errorcode[num] in names + class CopyItem: """""" def __init__(self, src, dst, vea_pre, cp_attr, cp_xattr, logger): #initialization @@ -60,18 +66,20 @@ class CopyItem: def eval(self): #access to the extended attributes present in some operating systems/filesystems by xattr try: - self.vuuid = xattr.get(self.src, self.vea_pre + 'ltfs.volumeUUID') + self.vuuid = xattr.getxattr(self.src, self.vea_pre + 'ltfs.volumeUUID') except Exception as e: self.vuuid = '' return (self.vuuid, self.part, self.start) try: - self.part = xattr.get(self.src, self.vea_pre + 'ltfs.partition') - start_str = xattr.get(self.src, self.vea_pre + 'ltfs.startblock') + self.part = xattr.getxattr(self.src, self.vea_pre + 'ltfs.partition') + start_str = xattr.getxattr(self.src, self.vea_pre + 'ltfs.startblock') self.start = int(start_str) self.size = os.path.getsize(self.src) except Exception as e: self.logger.error('Failed to get attribute of "{0}": {1}'.format(self.src, str(str(e)))) + self.part = b'a' + self.start = 0 return (self.vuuid, self.part, self.start) @@ -87,12 +95,12 @@ class CopyItem: if self.cp_xattr: # Capture EAs of the source file src_attributes = {} - for key in xattr.list(self.src): - src_attributes[key] = xattr.get(self.src, key) + for key in xattr.listxattr(self.src): + src_attributes[key] = xattr.getxattr(self.src, key) # Set EAs of the destination file (_, filename) = os.path.split(self.src) for key in src_attributes: - xattr.set(os.path.join(self.dst, filename), key, src_attributes[key]) + xattr.setxattr(self.dst, key, src_attributes[key]) else: #Only copy data shutil.copy(self.src, self.dst) except Exception as e: @@ -176,7 +184,7 @@ class CopyQueue: self.logger.log(NOTSET + 1, 'Making a directory {}'.format(d)) os.mkdir(d) except OSError as e: - if e.errno != 17: # Not EEXIST + if e.errno != errno.EEXIST: self.logger.error(str(e) + "\n") exit(1) @@ -249,7 +257,7 @@ LTFS_SIG_VEA='ltfs.softwareProduct' plat = platform.system() if plat == 'Linux': VEA_PREFIX='user.' -elif plat == 'Darwin': +elif plat == 'Darwin' or plat == 'FreeBSD': VEA_PREFIX='' else: sys.stderr.write("unsupported platform '{0}'\n".format(plat)) @@ -267,7 +275,7 @@ logger_debug = 6 parser = argparse.ArgumentParser(description = 'Copy files from source to destination with LTFS order optimization') parser.add_argument('SOURCE', help='source files', nargs='*') parser.add_argument('DEST', help='destination', nargs='?') -parser.add_argument('-p', help='preserve attributes with shutil.copy2()', action='store_true') +parser.add_argument('-p', help='preserve attributes with shutil.copy2(). Ignored if destination does not support xattr', action='store_true') parser.add_argument('-r', '--recursive', help='copy directories recursively', action='store_true') parser.add_argument('-t', '--target-directory', help='copy all SOURCE arguments into TARGET_DIRECTORY') #parser.add_argument('-z', '--zero', help='handle NULL delimited source list (assume input \'find -print0\')', action='store_true') @@ -332,12 +340,21 @@ if args.DEST == None: logger.error('No destination is specified') exit(2) +if args.keep_tree is None: + args.keep_tree = '' + # Special case: # Copy source is only one file if args.recursive == False and len(args.SOURCE) == 1: logger.log(NOTSET + 1, 'Single file copy') if os.path.isfile(args.SOURCE[0]): try: + if len(args.keep_tree): + dst = args.DEST + '/' + args.SOURCE[0][len(args.keep_tree):] + args.DEST = os.path.normpath(dst) + (new_d, t) = os.path.split(dst) + if not os.path.exists(new_d): + os.makedirs(new_d) shutil.copy(args.SOURCE[0], args.DEST) except Exception as e: logger.error(str(e)) @@ -351,17 +368,20 @@ if args.recursive == False and len(args.SOURCE) == 1: logger.log(NOTSET + 1, 'Checking destination is LTFS or not') direct_write_threads = 8 try: - sig = xattr.get(args.DEST, VEA_PREFIX + LTFS_SIG_VEA) + sig = xattr.getxattr(args.DEST, VEA_PREFIX + LTFS_SIG_VEA) - if sig.startswith("LTFS"): + if sig.startswith(b"LTFS"): logger.info("Destination {0} is LTFS".format(args.DEST)) direct_write_threads = 1 else: logger.info("Destination {0} is not LTFS\n".format(args.DEST)) except IOError as e: - if e.errno != 61 and e.errno != 93: # Not ENODATA, ENOATTR + if not is_errno(e.errno, ['ENODATA', 'ENOATTR', 'ENOTSUP', 'EOPNOTSUPP']): logger.error('Check destination (I/O):' + str(e)) exit(2) + if is_errno(e.errno, ['ENOTSUP', 'EOPNOTSUPP']) and args.p: + logger.warning("{0} does not support xattr. Cannot use -p. Attributes will not be preserved during copy.".format(args.DEST)) + args.p = False logger.warning("Destination {0} is not LTFS".format(args.DEST)) except Exception as e: logger.error('Check destination:' + str(e)) @@ -374,9 +394,6 @@ if len(args.SOURCE) == 0: args.SOURCE.append(line.rstrip('\r\n')) logger.log(NOTSET + 1, 'Source: {}'.format(args.SOURCE)) -if args.keep_tree is None: - args.keep_tree = '' - # Create the list of copy item copyq = CopyQueue(logger) for s in args.SOURCE: diff --git a/src/utils/ltfsck.c b/src/utils/ltfsck.c index d8e2b296..dc7cef85 100644 --- a/src/utils/ltfsck.c +++ b/src/utils/ltfsck.c @@ -3,7 +3,7 @@ ** OO_Copyright_BEGIN ** ** -** Copyright 2010, 2020 IBM Corp. All rights reserved. +** Copyright 2010, 2022 IBM Corp. All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions @@ -96,27 +96,27 @@ enum { }; struct other_check_opts { - struct config_file *config; /**< Configurate data read from the global LTFS config file. */ - char *devname; /**< Device to format */ - char *backend_path; /**< Path to tape backend shared library */ - char *kmi_backend_name; /**< Name or path to the key manager interface backend library */ - int op_mode; /**< Operation mode */ - int search_mode; /**< Search mode for index */ - char *str_gen; /**< Rollback point specified by command line (generation)*/ - unsigned int point_gen; /**< Rollback point (generation) */ - bool erase_history; /**< overwrite existing data at rollback */ - bool recover_blocks; /**< Recover unreferenced blocks at the ends of the partitions? */ - bool deep_recovery; /**< Recover EOD missing cartridge? */ - int verbosity; /**< Print extra messages? */ - char *prg_name; /**< Program name */ - bool quiet; /**< Suppress information messages */ - bool trace; /**< Generate debug output */ - bool syslogtrace; /**< Generate debug output to stderr and syslog*/ - bool fulltrace; /**< Trace function calls */ - int traverse_mode; /**< Traverse strategy for listing index */ - bool full_index_info; /**< Print full index infomation in list mode */ - bool capture_index; /**< Capture index in list mode */ - bool salvage_points; /**< List rollback points from no-EOD cartridge? */ + struct config_file *config; /**< Configurate data read from the global LTFS config file. */ + char *devname; /**< Device to format */ + char *backend_path; /**< Path to tape backend shared library */ + char *kmi_backend_name; /**< Name or path to the key manager interface backend library */ + int op_mode; /**< Operation mode */ + int search_mode; /**< Search mode for index */ + char *str_gen; /**< Rollback point specified by command line (generation)*/ + unsigned int point_gen; /**< Rollback point (generation) */ + bool erase_history; /**< overwrite existing data at rollback */ + bool recover_blocks; /**< Recover unreferenced blocks at the ends of the partitions? */ + bool deep_recovery; /**< Recover EOD missing cartridge? */ + int verbosity; /**< Print extra messages? */ + char *prg_name; /**< Program name */ + bool quiet; /**< Suppress information messages */ + bool trace; /**< Generate debug output */ + bool syslogtrace; /**< Generate debug output to stderr and syslog*/ + bool fulltrace; /**< Trace function calls */ + int traverse_mode; /**< Traverse strategy for listing index */ + bool full_index_info; /**< Print full index infomation in list mode */ + char *capture_dir; /**< Capture index in list mode and it's directory */ + bool salvage_points; /**< List rollback points from no-EOD cartridge? */ }; struct index_info @@ -167,7 +167,7 @@ static struct option long_options[] = { {"generation", 1, 0, 'g'}, {"traverse", 1, 0, 'v'}, {"kmi-backend", 1, 0, '-'}, - {"capture-index", 0, 0, '+'}, + {"capture-index", 1, 0, '+'}, {"rollback" , 0, 0, 'r'}, {"no-rollback" , 0, 0, 'n'}, {"full-recovery" , 0, 0, 'f'}, @@ -350,7 +350,7 @@ int main(int argc, char **argv) break; case '+': opt.op_mode = MODE_LIST_POINT; - opt.capture_index = true; + opt.capture_dir = strdup(optarg); break; case 'r': opt.op_mode = MODE_ROLLBACK; @@ -711,6 +711,7 @@ int check_ltfs_volume(struct ltfs_volume *vol, struct other_check_opts *opt) return LTFSCK_UNCORRECTED; } else { print_criteria_info(vol); + ltfs_set_commit_message_reason(SYNC_CHECK, vol); ltfs_unmount(SYNC_CHECK, vol); ltfsmsg(LTFS_INFO, 16022I); return LTFSCK_CORRECTED; @@ -840,7 +841,8 @@ void _print_index_header(bool full_info) void _print_index(struct ltfs_volume *vol, struct index_info *list, struct other_check_opts *opt) { struct tm *t_st; - int i; + char *new_path = NULL; + int i, ret; if(!opt) return; @@ -935,8 +937,26 @@ void _print_index(struct ltfs_volume *vol, struct index_info *list, struct other else printf(" No commit message\n"); - if (opt->capture_index) - ltfs_save_index_to_disk(".", NULL, true, vol); + /* Rename reading xml to official name */ + if (vol->index_cache_path_r && opt->capture_dir) { + if (HAVE_BARCODE(vol)) + ret = asprintf(&new_path, "%s/%s-%d-%c.schema", opt->capture_dir, vol->label->barcode, + vol->index->generation, list->selfptr.partition); + else + ret = asprintf(&new_path, "%s/%s-%d-%c.schema", opt->capture_dir, vol->label->vol_uuid, + vol->index->generation, list->selfptr.partition); + + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10001E, "_print_index: path"); + return; + } + + ret = rename(vol->index_cache_path_r, new_path); + if (ret < 0) { + ltfsmsg(LTFS_WARN, 16112W, vol->index_cache_path_r, new_path, errno); + } + free(new_path); + } return; } @@ -968,7 +988,7 @@ void print_index_array(struct ltfs_volume *vol, struct index_info *list, void *o while (cur) { _print_index(vol, cur, opt); - cur = cur-> next; + cur = cur->next; } return; @@ -1134,7 +1154,7 @@ int _rollback_ip(struct ltfs_volume *vol, struct other_check_opts *opt, struct t if (ret != LTFSCK_NO_ERRORS) ltfsmsg(LTFS_ERR, 16059E, ret); } else { - ret = ltfs_write_index(ltfs_ip_id(vol), SYNC_ROLLBACK, vol); + ret = ltfs_write_index(ltfs_ip_id(vol), SYNC_ROLLBACK, LTFS_FULL_INDEX, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 16060E, ret); ret = LTFSCK_OPERATIONAL_ERROR; @@ -1155,7 +1175,8 @@ int _rollback_dp(struct ltfs_volume *vol, struct other_check_opts *opt, struct t if (ret != LTFSCK_NO_ERRORS) ltfsmsg(LTFS_ERR, 16055E, ret); } else { - ret = ltfs_write_index(ltfs_dp_id(vol), SYNC_ROLLBACK, vol); + ltfs_set_commit_message_reason(SYNC_ROLLBACK, vol); + ret = ltfs_write_index(ltfs_dp_id(vol), SYNC_ROLLBACK, LTFS_FULL_INDEX, vol); if (ret < 0) { ltfsmsg(LTFS_ERR, 16056E, ret); ret = LTFSCK_OPERATIONAL_ERROR; @@ -1269,6 +1290,7 @@ int rollback(struct ltfs_volume *vol, struct other_check_opts *opt) r.current_pos = ltfs_get_index_selfpointer(vol); ltfsmsg(LTFS_DEBUG, 16081D, ltfs_get_index_generation(vol), (int)r.current_pos.partition, (unsigned long long)r.current_pos.block); + ltfs_set_commit_message_reason(SYNC_ROLLBACK, vol); ltfs_unmount(SYNC_ROLLBACK, vol); vol->index = NULL; } @@ -1280,11 +1302,11 @@ int rollback(struct ltfs_volume *vol, struct other_check_opts *opt) } /* Find target index */ - ret = ltfs_traverse_index_backward(vol, ltfs_ip_id(vol), opt->point_gen, + ret = ltfs_traverse_index_backward(vol, ltfs_ip_id(vol), opt->point_gen, false, search_index_by_gen, (void *)(&(r.target_info)), (void *)opt); if (ret == -LTFS_NO_INDEX) { if (opt->erase_history) { - ret = ltfs_traverse_index_forward(vol, ltfs_dp_id(vol), opt->point_gen, + ret = ltfs_traverse_index_forward(vol, ltfs_dp_id(vol), opt->point_gen, false, search_index_by_gen, (void *)(&(r.target_info)), (void *)opt); if (ret == -LTFS_NO_INDEX) { ltfsmsg(LTFS_ERR, 16072E, ret); @@ -1294,7 +1316,7 @@ int rollback(struct ltfs_volume *vol, struct other_check_opts *opt) return LTFSCK_OPERATIONAL_ERROR;; } } else { - ret = ltfs_traverse_index_backward(vol, ltfs_dp_id(vol), opt->point_gen, + ret = ltfs_traverse_index_backward(vol, ltfs_dp_id(vol), opt->point_gen, false, search_index_by_gen, (void *)(&(r.target_info)), (void *)opt); if (ret != LTFSCK_NO_ERRORS) { ltfsmsg(LTFS_ERR, 16072E, ret); @@ -1352,7 +1374,7 @@ int rollback(struct ltfs_volume *vol, struct other_check_opts *opt) int list_rollback_points_normal(struct ltfs_volume *vol, struct other_check_opts *opt) { - int ret = LTFSCK_NO_ERRORS; + int ret = LTFSCK_NO_ERRORS, fd = -1; /* Load tape and read labels */ ret = load_tape(vol); @@ -1372,14 +1394,41 @@ int list_rollback_points_normal(struct ltfs_volume *vol, struct other_check_opts } } + /* Configure index capturing */ + if (opt->capture_dir) { + /* Confirm the directory and construct read cache path */ + ret = asprintf(&vol->index_cache_path_r, "%s/reading_index.xml", opt->capture_dir); + + if (ret > 0) { + fd = open(vol->index_cache_path_r, O_WRONLY | O_BINARY | O_CREAT, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); + if (fd < 0) { + ltfsmsg(LTFS_WARN, 16113W, opt->capture_dir, errno); + free(vol->index_cache_path_r); + vol->index_cache_path_r = NULL; + } else { + ltfsmsg(LTFS_INFO, 16114I, opt->capture_dir); + close(fd); + fd = -1; + } + } else { + ltfsmsg(LTFS_ERR, 10001E, "capture_dir"); + ltfs_volume_free(&vol); + return 1; + } + } else { + ltfsmsg(LTFS_INFO, 16115I); + } + + /* Print header */ _print_index_header(opt->full_index_info); /* read index from the index partition */ if(opt->traverse_mode == TRAVERSE_FORWARD) - ret = ltfs_traverse_index_forward(vol, ltfs_ip_id(vol), opt->point_gen, + ret = ltfs_traverse_index_forward(vol, ltfs_ip_id(vol), opt->point_gen, true, print_a_index_noheader, NULL, (void *)opt); else - ret = ltfs_traverse_index_backward(vol, ltfs_ip_id(vol), opt->point_gen, + ret = ltfs_traverse_index_backward(vol, ltfs_ip_id(vol), opt->point_gen, true, print_a_index_noheader, NULL, (void *)opt); if (ret != LTFSCK_NO_ERRORS) { ltfsmsg(LTFS_ERR, 16075E, ret); @@ -1388,10 +1437,10 @@ int list_rollback_points_normal(struct ltfs_volume *vol, struct other_check_opts /* read index from the data partition */ if(opt->traverse_mode == TRAVERSE_FORWARD) - ret = ltfs_traverse_index_forward(vol, ltfs_dp_id(vol), opt->point_gen, + ret = ltfs_traverse_index_forward(vol, ltfs_dp_id(vol), opt->point_gen, true, print_a_index_noheader, NULL, (void *)opt); else - ret = ltfs_traverse_index_backward(vol, ltfs_dp_id(vol), opt->point_gen, + ret = ltfs_traverse_index_backward(vol, ltfs_dp_id(vol), opt->point_gen, true, print_a_index_noheader, NULL, (void *)opt); if (ret != LTFSCK_NO_ERRORS) { ltfsmsg(LTFS_ERR, 16076E, ret); @@ -1427,7 +1476,7 @@ int list_rollback_points_no_eod(struct ltfs_volume *vol, struct other_check_opts /* Read index from the data partition */ /* We don't need to read IP because index in DP is always newer (or same) than IP in case of WORM */ - ret = ltfs_traverse_index_no_eod(vol, ltfs_dp_id(vol), opt->point_gen, + ret = ltfs_traverse_index_no_eod(vol, ltfs_dp_id(vol), opt->point_gen, true, print_a_index_noheader, NULL, (void *)opt); if (ret != LTFSCK_NO_ERRORS) { ltfsmsg(LTFS_ERR, 16076E, ret); @@ -1487,7 +1536,7 @@ int _ltfsck_validate_options(struct other_check_opts *opt) else if (opt->traverse_mode == TRAVERSE_BACKWARD) ltfsmsg(LTFS_INFO, 16084I); - if (opt->capture_index && opt->search_mode == SEARCH_BY_GEN) { + if (opt->capture_dir && opt->search_mode == SEARCH_BY_GEN) { if (!opt->str_gen) { ltfsmsg(LTFS_ERR, 16004E); return LTFSCK_USAGE_SYNTAX_ERROR; diff --git a/src/utils/ltfsindextool.c b/src/utils/ltfsindextool.c new file mode 100644 index 00000000..c4d47a64 --- /dev/null +++ b/src/utils/ltfsindextool.c @@ -0,0 +1,775 @@ +/* +** +** OO_Copyright_BEGIN +** +** +** Copyright 2010, 2020 IBM Corp. All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. Neither the name of the copyright holder nor the names of its +** contributors may be used to endorse or promote products derived from +** this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +** AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +** IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +** ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +** LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +** CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +** SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +** INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +** CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +** ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +** POSSIBILITY OF SUCH DAMAGE. +** +** +** OO_Copyright_END +** +************************************************************************************* +** +** COMPONENT NAME: IBM Linear Tape File System +** +** FILE NAME: utils/ltfsindextool.c +** +** DESCRIPTION: The low level index tool +** +** AUTHORS: Atsushi Abe +** piste.2750@gmail.com +** +************************************************************************************* +*/ + +#ifdef mingw_PLATFORM +#include "libltfs/arch/win/win_util.h" +#else +#include +#endif /* mingw_PLATFORM */ + +#include +#include "libltfs/ltfs_fuse_version.h" +#include + +#include "libltfs/ltfs.h" +#include "libltfs/xml_libltfs.h" +#include "ltfs_copyright.h" +#include "libltfs/plugin.h" +#include "libltfs/kmi.h" +#include "libltfs/tape.h" + +volatile char *copyright = LTFS_COPYRIGHT_0"\n"LTFS_COPYRIGHT_1"\n"LTFS_COPYRIGHT_2"\n" \ + LTFS_COPYRIGHT_3"\n"LTFS_COPYRIGHT_4"\n"LTFS_COPYRIGHT_5"\n"; + +#ifdef __APPLE__ +#include "libltfs/arch/osx/osx_string.h" +#endif + +#ifdef mingw_PLATFORM +char *bin_ltfsindextool_dat; +#else +extern char bin_ltfsindextool_dat[]; +#endif + +typedef enum { + OP_CHECK, + OP_CAPTURE, +} operation_mode_t; + +struct indextool_opts { + operation_mode_t mode; /**< Operation mode */ + char *filename; /**< Index filename to check in check mode */ + char *devname; /**< Device name to capture index in capture mode */ + int partition; /**< partition to operate */ + uint64_t start_pos; /**< start position */ + char *out_dir; /**< Output dir for captured indexes */ + unsigned long blocksize; /**< Nominal tape block size */ + struct config_file *config; /**< Configuration data read from the global LTFS config file */ + char *backend_path; /**< Path to tape backend shared library */ + char *kmi_backend_name; /**< Name or path to the key manager interface backend library */ + bool quiet; /**< Quiet mode indicator */ + bool trace; /**< Debug mode indicator */ + bool syslogtrace; /**< Generate debug output to stderr and syslog*/ +}; + +/* Command line options */ +#define PART_BOTH (-1) +#define START_POS (5) +#define OUTPUT_DIR "." +#define KEY_MAX_OFFSET (0x30) + +static const char *short_options = "i:e:d:p:s:o:b:qthV"; +static struct option long_options[] = { + {"config", 1, 0, 'i'}, + {"backend", 1, 0, 'e'}, + {"device", 1, 0, 'd'}, + {"partition", 1, 0, 'p'}, + {"start-pos", 1, 0, 's'}, + {"output-dir", 1, 0, '^'}, + {"blocksize", 1, 0, 'b'}, + {"kmi-backend", 1, 0, '-'}, + {"quiet", 0, 0, 'q'}, + {"trace", 0, 0, 't'}, + {"syslogtrace", 0, 0, '!'}, + {"help", 0, 0, 'h'}, + {"version", 0, 0, 'V'}, + {0, 0, 0, 0} +}; + +/* Private functions */ +static inline int _open_output_file(tape_partition_t part, + tape_block_t start_pos, + char *base_path) +{ + int ret; + char *fname = NULL; + + ret = asprintf(&fname, "%s/ltfs-index-%u-%llu.xml", base_path, + (unsigned int) part, (unsigned long long)start_pos); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19532E); + return -1; + } + + ltfsmsg(LTFS_INFO, 19547I, fname); + ret = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19533E, fname, errno); + } + + free(fname); + + return ret; +} + +static inline void _close_output_file(int fd) +{ + fsync(fd); + close(fd); +} + +/** Capture indexes on the partition. + * + */ +static int ltfs_capture_index_raw(tape_partition_t part, + tape_block_t start_pos, + int blocksize, + char *base_path, + struct ltfs_volume *vol) +{ + int ret = 0, fd = -1; + ssize_t nread, nwrite, index_len = 0; + struct tc_position pos; + char *buf = NULL, *key = NULL, check_buf[KEY_MAX_OFFSET + 1]; + + pos.partition = part; + pos.block = start_pos; + + buf = malloc(blocksize); + if (!buf) { + ltfsmsg(LTFS_ERR, 19516E); + return -LTFS_NO_MEMORY; + } + + ret = tape_seek(vol->device, &pos); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19517E, (unsigned int)part, (unsigned long long)start_pos, ret); + return ret; + } + + while (ret == 0) { + index_len = 0; + + ret = tape_get_position(vol->device, &pos); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19518E, ret); + break; + } + + nread = tape_read(vol->device, buf, blocksize, true, vol->kmi_handle); + if (nread < 0) { + ltfsmsg(LTFS_ERR, 19519E, + (unsigned int)pos.partition, (unsigned long long)pos.block, nread); + ret = nread; + break; + } + pos.block++; + + memset(check_buf, 0x00, KEY_MAX_OFFSET + 1); + strncpy(check_buf, buf, KEY_MAX_OFFSET); + key = strstr(check_buf, "device, buf, blocksize, true, vol->kmi_handle); + if (nread == -EDEV_EOD_DETECTED) { + ret = nread; + ltfsmsg(LTFS_ERR, 19538E, + (unsigned int)pos.partition, + (unsigned long long)(pos.block)); + ltfsmsg(LTFS_INFO, 19539I, (unsigned long long)index_len); + break; + } + + if (nread > 0) { + /* Write a block to the file */ + nwrite = write(fd, buf, nread); + if (nwrite == nread) { + index_len += nread; + } else { + ltfsmsg(LTFS_ERR, 19536E, nwrite, errno); + _close_output_file(fd); + return -LTFS_CACHE_IO; + } + pos.block++; + } else if (!nread) { + /* Detect a FM (the end of the index), do nothing */ + ltfsmsg(LTFS_INFO, 19537I, + (unsigned int)pos.partition, + (unsigned long long)(pos.block)); + ltfsmsg(LTFS_INFO, 19539I, (unsigned long long)index_len); + break; + } else { + ltfsmsg(LTFS_ERR, 19519E, + (unsigned int)pos.partition, (unsigned long long)pos.block, nread); + ret = nread; + break; + } + } + + _close_output_file(fd); + } else { + /* seek to next FM */ + if (key) + ltfsmsg(LTFS_INFO, 19530I, + (unsigned int)pos.partition, + (unsigned long long)(pos.block - 1), + (int)(key - buf)); + else + ltfsmsg(LTFS_INFO, 19530I, + (unsigned int)pos.partition, + (unsigned long long)(pos.block - 1), + (int)0); + + /* Do nothig at (nread == 0) because tape hits a FM */ + if (nread > 0) { + ret = tape_spacefm(vol->device, 1); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19531E, (unsigned int)part, (unsigned long long)start_pos, ret); + break; + } + } + } + } + + if (ret == -EDEV_EOD_DETECTED) { + ret = tape_get_position(vol->device, &pos); + if (!ret) { + ltfsmsg(LTFS_INFO, 19534I,(unsigned int)pos.partition, (unsigned long long)pos.block); + } else { + ltfsmsg(LTFS_INFO, 19535I, ret); + } + ret = 0; + } + + if (buf) + free(buf); + + return ret; +} + +static int _capture(struct indextool_opts *opt, struct ltfs_volume *vol) +{ + int r, ret = 0; + tape_partition_t p; + + if (opt->partition == PART_BOTH) { + ltfsmsg(LTFS_INFO, 19504I); + for (p = 0; p < 2; p++) { + r = ltfs_capture_index_raw(p, 5, opt->blocksize, opt->out_dir, vol); + if (!ret) + ret = r; + } + } else { + ltfsmsg(LTFS_INFO, 19505I, (unsigned int)opt->partition, (unsigned long long)opt->start_pos); + ret = ltfs_capture_index_raw(opt->partition, opt->start_pos, opt->blocksize, opt->out_dir, vol); + } + + return ret; +} + +static int _indextool_validate_options(char *prg_name, struct indextool_opts *opt) +{ + ltfsmsg(LTFS_DEBUG, 19525D); + + /* Validate filename and devname and decide a operation mode*/ + if (opt->filename) { + opt->mode = OP_CHECK; + } else if (opt->devname) { + opt->mode = OP_CAPTURE; + } else { + ltfsmsg(LTFS_ERR, 19526E); + return 1; + } + + /* Validate partition */ + if (opt->partition != PART_BOTH && opt->partition != 0 && opt->partition != 1) { + ltfsmsg(LTFS_ERR, 19540E); + return 1; + } + + /* Validate start position */ + if (opt->start_pos < START_POS) { + ltfsmsg(LTFS_ERR, 19548E, (unsigned long long)opt->start_pos); + return 1; + } + + ltfsmsg(LTFS_DEBUG, 19527D); + return 0; +} + +static int check_index(struct ltfs_volume *vol, struct indextool_opts *opt, void *args) +{ + int ret = 0; + + ltfsmsg(LTFS_INFO, 19543I, opt->filename); + + vol->label->blocksize = opt->blocksize; + ret = xml_schema_from_file(opt->filename, vol->index, vol); + + if (!ret) { + ltfsmsg(LTFS_INFO, 19544I); + } else { + ltfsmsg(LTFS_ERR, 19545E, ret); + } + + return ret; +} + +static int capture_index(struct ltfs_volume *vol, struct indextool_opts *opt, void *args) +{ + int ret = INDEXTOOL_OPERATIONAL_ERROR; + struct libltfs_plugin backend; /* tape driver backend */ + struct libltfs_plugin kmi; /* key manager interface backend */ + + /* load the backend, open the tape device, and load a tape */ + ltfsmsg(LTFS_DEBUG, 19506D); + ret = plugin_load(&backend, "tape", opt->backend_path, opt->config); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19508E, opt->backend_path); + return INDEXTOOL_OPERATIONAL_ERROR; + } + if (opt->kmi_backend_name) { + ret = plugin_load(&kmi, "kmi", opt->kmi_backend_name, opt->config); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19509E, opt->kmi_backend_name); + return INDEXTOOL_OPERATIONAL_ERROR; + } + } + ret = ltfs_device_open(opt->devname, backend.ops, vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19510E, opt->devname, ret); + ret = INDEXTOOL_OPERATIONAL_ERROR; + goto out_unload_backend; + } + ret = ltfs_parse_tape_backend_opts(args, vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19513E); + goto out_unload_backend; + } + if (opt->kmi_backend_name) { + ret = kmi_init(&kmi, vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19511E, opt->devname, ret); + goto out_unload_backend; + } + + ret = ltfs_parse_kmi_backend_opts(args, vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19512E); + goto out_unload_backend; + } + + ret = tape_clear_key(vol->device, vol->kmi_handle); + if (ret < 0) + goto out_unload_backend; + } + + { + int i = 0; + struct fuse_args *a = args; + + for (i = 0; i < a->argc && a->argv[i]; ++i) { + if (!strcmp(a->argv[i], "-o")) { + ltfsmsg(LTFS_ERR, 19514E, a->argv[i], a->argv[i + 1] ? a->argv[i + 1] : ""); + ret = INDEXTOOL_USAGE_SYNTAX_ERROR; + goto out_unload_backend; + } + } + } + + ltfs_load_tape(vol); + ret = ltfs_wait_device_ready(vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19515E); + ret = INDEXTOOL_OPERATIONAL_ERROR; + goto out_close; + } + + vol->append_only_mode = false; + vol->set_pew = false; + ret = ltfs_setup_device(vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19515E); + ret = INDEXTOOL_OPERATIONAL_ERROR; + goto out_close; + } + ltfsmsg(LTFS_DEBUG, 19507D); + + /* Capture_index */ + ret = _capture(opt, vol); + + /* close the tape device and unload the backend */ + ltfsmsg(LTFS_DEBUG, 19520D); + +out_close: + ltfs_device_close(vol); + ltfs_volume_free(&vol); + ltfs_unset_signal_handlers(); + if (ret == INDEXTOOL_NO_ERRORS) + ltfsmsg(LTFS_DEBUG, 19522D); +out_unload_backend: + if (ret == INDEXTOOL_NO_ERRORS) { + ret = plugin_unload(&backend); + if (ret < 0) + ltfsmsg(LTFS_WARN, 19521W); + if (opt->kmi_backend_name) { + ret = plugin_unload(&kmi); + if (ret < 0) + ltfsmsg(LTFS_WARN, 19528W); + } + ret = INDEXTOOL_NO_ERRORS; + } else { + plugin_unload(&backend); + if (opt->kmi_backend_name) + plugin_unload(&kmi); + } + + if (ret == INDEXTOOL_NO_ERRORS) + ltfsmsg(LTFS_INFO, 19524I); + else + ltfsmsg(LTFS_INFO, 19523I, ret); + + return ret; +} + +static void show_usage(char *appname, struct config_file *config, bool full) +{ + struct libltfs_plugin backend; + const char *default_backend; + char *devname = NULL; + + default_backend = config_file_get_default_plugin("tape", config); + if (default_backend && plugin_load(&backend, "tape", default_backend, config) == 0) { + devname = strdup(ltfs_default_device_name(backend.ops)); + plugin_unload(&backend); + } + + if (! devname) + devname = strdup(""); + + fprintf(stderr, "\n"); + ltfsresult(19900I, appname); /* Usage: %s */ + fprintf(stderr, "\n"); + ltfsresult(19901I); /* Available options are: */ + ltfsresult(19902I); /* -d, --device= */ + ltfsresult(19903I); /* -p, --partition=<0|1> */ + ltfsresult(19904I, START_POS); /* -s, --start-pos */ + ltfsresult(19905I, OUTPUT_DIR); /* -output-dir */ + ltfsresult(19906I, LTFS_DEFAULT_BLOCKSIZE); /* -b, --blocksize */ + ltfsresult(19907I, LTFS_CONFIG_FILE); /* -i, --config= */ + ltfsresult(19908I, default_backend); /* -e, --backend */ + ltfsresult(19909I, config_file_get_default_plugin("kmi", config)); /* --kmi-backend */ + ltfsresult(19910I); /* -q, --quiet */ + ltfsresult(19911I); /* -t, --trace */ + ltfsresult(19912I); /* -V, --version */ + ltfsresult(19913I); /* -h, --help */ + fprintf(stderr, "\n"); + plugin_usage(appname, "driver", config); + fprintf(stderr, "\n"); + plugin_usage(appname, "kmi", config); + fprintf(stderr, "\n"); + ltfsresult(19914I); /* Usage example: */ + ltfsresult(19915I, appname, devname, 0); + free(devname); +} + +/* Main routine */ +int main(int argc, char **argv) +{ + struct ltfs_volume *vol; + struct indextool_opts opt; + int ret, log_level, syslog_level, i, cmd_args_len; + char *lang, *cmd_args; + const char *config_file = NULL; + void *message_handle; + + int fuse_argc = argc; + char **fuse_argv = calloc(fuse_argc, sizeof(char *)); + if (! fuse_argv) { + return INDEXTOOL_OPERATIONAL_ERROR; + } + for (i = 0; i < fuse_argc; ++i) { + fuse_argv[i] = strdup(argv[i]); + if (! fuse_argv[i]) { + return INDEXTOOL_OPERATIONAL_ERROR; + } + } + struct fuse_args args = FUSE_ARGS_INIT(fuse_argc, fuse_argv); + + /* Check for LANG variable and set it to en_US.UTF-8 if it is unset. */ + lang = getenv("LANG"); + if (! lang) { + fprintf(stderr, "LTFS9015W Setting the locale to 'en_US.UTF-8'. If this is wrong, please set the LANG environment variable before starting mkltfs.\n"); + ret = setenv("LANG", "en_US.UTF-8", 1); + if (ret) { + fprintf(stderr, "LTFS9016E Cannot set the LANG environment variable\n"); + return INDEXTOOL_OPERATIONAL_ERROR; + } + } + + /* Start up libltfs with the default logging level. */ +#ifndef mingw_PLATFORM + openlog("ltfsindextool", LOG_PID, LOG_USER); +#endif + ret = ltfs_init(LTFS_INFO, true, false); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10000E, ret); + return INDEXTOOL_OPERATIONAL_ERROR; + } + + /* Setup signal handler to terminate cleanly */ + ret = ltfs_set_signal_handlers(); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10013E); + return INDEXTOOL_OPERATIONAL_ERROR; + } + + /* Register messages with libltfs */ + ret = ltfsprintf_load_plugin("bin_ltfsindextool", bin_ltfsindextool_dat, &message_handle); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10012E, ret); + return INDEXTOOL_OPERATIONAL_ERROR; + } + + /* Set up empty options and load the configuration file. */ + memset(&opt, 0, sizeof(struct indextool_opts)); + opt.blocksize = LTFS_DEFAULT_BLOCKSIZE; + opt.partition = PART_BOTH; + opt.start_pos = START_POS; + opt.out_dir = OUTPUT_DIR; + + /* Check for a config file path given on the command line */ + while (true) { + int option_index = 0; + int c = getopt_long(argc, argv, short_options, long_options, &option_index); + if (c == -1) + break; + if (c == 'i') { + config_file = strdup(optarg); + break; + } + } + + /* Load configuration file */ + ret = config_file_load(config_file, &opt.config); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 10008E, ret); + return INDEXTOOL_OPERATIONAL_ERROR; + } + + /* Parse all command line arguments */ + + optind = 1; + int num_of_o = 0; + while (true) { + int option_index = 0; + int c = getopt_long(argc, argv, short_options, long_options, &option_index); + if (c == -1) + break; + + switch (c) { + case 'i': + break; + case 'e': + free(opt.backend_path); + opt.backend_path = strdup(optarg); + break; + case 'd': + opt.devname = strdup(optarg); + break; + case 'p': + opt.partition = atoi(optarg); + break; + case 's': + opt.start_pos = strtoull(optarg, NULL, 0); + break; + case '^': + opt.out_dir = strdup(optarg); + break; + case 'b': + opt.blocksize = atoi(optarg); + break; + case '-': + opt.kmi_backend_name = strdup(optarg); + break; + case 'q': + opt.quiet = true; + break; + case 't': + opt.trace = true; + break; + case '!': + opt.syslogtrace = true; + break; + case 'h': + show_usage(argv[0], opt.config, false); + return 0; + case 'V': + ltfsresult(19546I, "ltfsindextool", PACKAGE_VERSION); + ltfsresult(19546I, "LTFS Format Specification", LTFS_INDEX_VERSION_STR); + return 0; + case '?': + default: + show_usage(argv[0], opt.config, false); + return INDEXTOOL_USAGE_SYNTAX_ERROR; + } + } + + if(argv[optind + num_of_o]) + opt.filename = strdup(argv[optind + num_of_o]); + + if(_indextool_validate_options(argv[0], &opt)) { + return PROG_USAGE_SYNTAX_ERROR; + } + + /* Pick up default backend if one wasn't specified before */ + if (! opt.backend_path) { + const char *default_backend = config_file_get_default_plugin("tape", opt.config); + if (! default_backend) { + ltfsmsg(LTFS_ERR, 10009E); + return INDEXTOOL_OPERATIONAL_ERROR; + } + opt.backend_path = strdup(default_backend); + } + if (! opt.kmi_backend_name) { + const char *default_backend = config_file_get_default_plugin("kmi", opt.config); + if (default_backend) + opt.kmi_backend_name = strdup(default_backend); + else + opt.kmi_backend_name = strdup("none"); + } + if (opt.kmi_backend_name && strcmp(opt.kmi_backend_name, "none") == 0) + opt.kmi_backend_name = NULL; + + /* Set the logging level */ + if (opt.quiet && opt.trace) { + ltfsmsg(LTFS_ERR, 9012E); + show_usage(argv[0], opt.config, false); + return 1; + } else if (opt.quiet) { + log_level = LTFS_WARN; + syslog_level = LTFS_NONE; + } else if (opt.trace) { + log_level = LTFS_DEBUG; + syslog_level = LTFS_NONE; + } else if (opt.syslogtrace) + log_level = syslog_level = LTFS_DEBUG; + else { + log_level = LTFS_INFO; + syslog_level = LTFS_NONE; + } + + ltfs_set_log_level(log_level); + ltfs_set_syslog_level(syslog_level); + + /* Starting ltfsindextool */ + ltfsmsg(LTFS_INFO, 19500I, PACKAGE_NAME, PACKAGE_VERSION, log_level); + + /* Show command line arguments */ + for (i = 0, cmd_args_len = 0 ; i < argc; i++) { + cmd_args_len += strlen(argv[i]) + 1; + } + cmd_args = calloc(1, cmd_args_len + 1); + if (!cmd_args) { + /* Memory allocation failed */ + ltfsmsg(LTFS_ERR, 10001E, "ltfsindextool (arguments)"); + return INDEXTOOL_OPERATIONAL_ERROR; + } + strcat(cmd_args, argv[0]); + for (i = 1; i < argc; i++) { + strcat(cmd_args, " "); + strcat(cmd_args, argv[i]); + } + ltfsmsg(LTFS_INFO, 19542I, cmd_args); + free(cmd_args); + + /* Show build time information */ + ltfsmsg(LTFS_INFO, 19502I, BUILD_SYS_FOR); + ltfsmsg(LTFS_INFO, 19503I, BUILD_SYS_GCC); + + /* Show run time information */ + show_runtime_system_info(); + + /* Actually mkltfs logic starts here */ + ret = ltfs_volume_alloc("dummy", &vol); + if (ret < 0) { + ltfsmsg(LTFS_ERR, 19501E); + return INDEXTOOL_OPERATIONAL_ERROR; + } + + switch (opt.mode) { + case OP_CHECK: + ret = check_index(vol, &opt, &args); + break; + case OP_CAPTURE: + ret = capture_index(vol, &opt, &args); + break; + default: + ltfsmsg(LTFS_ERR, 19541E); + ret = PROG_USAGE_SYNTAX_ERROR; + break; + } + + /* Cleaning up */ + free(opt.backend_path); + free(opt.kmi_backend_name); + free(opt.devname); + config_file_free(opt.config); + ltfsprintf_unload_plugin(message_handle); + ltfs_finish(); + + return ret; +} diff --git a/src/utils/mkltfs.c b/src/utils/mkltfs.c index 208f8e55..75b5a03d 100644 --- a/src/utils/mkltfs.c +++ b/src/utils/mkltfs.c @@ -105,7 +105,8 @@ struct other_format_opts { bool trace; /**< Debug mode indicator */ bool syslogtrace; /**< Generate debug output to stderr and syslog*/ bool fulltrace; /**< Full trace mode indicator */ - bool long_wipe; /**< Clean up whole tape by long erase? */ + bool long_wipe; /**< Clean up whole tape by long erase? */ + bool destructive; /**< Use destructive FORMAT_MEDIUM or not */ }; /* Forward declarations */ @@ -130,6 +131,7 @@ static struct option long_options[] = { {"keep-capacity", 0, 0, 'k'}, {"wipe", 0, 0, 'w'}, {"long-wipe", 0, 0, '+'}, + {"destructive", 0, 0, '&'}, {"force", 0, 0, 'f'}, {"quiet", 0, 0, 'q'}, {"trace", 0, 0, 't'}, @@ -182,6 +184,7 @@ void show_usage(char *appname, struct config_file *config, bool full) ltfsresult(15419I); /* -k, --keep-capacity */ ltfsresult(15417I); /* -x, --fulltrace */ ltfsresult(15424I); /* --long-wipe */ + ltfsresult(15425I); /* --destructive */ fprintf(stderr, "\n"); plugin_usage(appname, "driver", config); fprintf(stderr, "\n"); @@ -265,6 +268,7 @@ int main(int argc, char **argv) opt.quiet = false; opt.blocksize = LTFS_DEFAULT_BLOCKSIZE; opt.long_wipe = false; + opt.destructive = false; /* Check for a config file path given on the command line */ while (true) { @@ -342,6 +346,9 @@ int main(int argc, char **argv) opt.unformat = true; opt.long_wipe = true; break; + case '&': + opt.destructive = true; + break; case 'q': opt.quiet = true; break; @@ -669,7 +676,7 @@ int format_tape(struct ltfs_volume *vol, struct other_format_opts *opt, void *ar /* Create partitions and write labels and indices to the tape */ ltfsmsg(LTFS_INFO, 15010I, DATA_PART_ID, DATA_PART_NUM); ltfsmsg(LTFS_INFO, 15011I, INDEX_PART_ID, INDEX_PART_NUM); - ret = ltfs_format_tape(vol, 0); + ret = ltfs_format_tape(vol, 0, opt->destructive); if (ret < 0) { if (ret == -LTFS_INTERRUPTED) { ltfsmsg(LTFS_ERR, 15045E); @@ -814,7 +821,7 @@ int unformat_tape(struct ltfs_volume *vol, struct other_format_opts *opt, void * ltfsmsg(LTFS_DEBUG, 15007D); /* Create 1 partition cartridge */ - ret = ltfs_unformat_tape(vol, opt->long_wipe); + ret = ltfs_unformat_tape(vol, opt->long_wipe, opt->destructive); if (ret < 0) { if (ret == -LTFS_INTERRUPTED) { ltfsmsg(LTFS_ERR, 15046E);